walletdb: add sync state object.

lib/wallet/records.js

@@ -12,6 +12,74 @@ var constants = require('../protocol/constants');
 var BufferReader = require('../utils/reader');
 var BufferWriter = require('../utils/writer');
 
+/**
+ * Wallet Tip
+ * @constructor
+ * @param {Hash} hash
+ * @param {Number} height
+ */
+
+function SyncState() {
+  if (!(this instanceof SyncState))
+    return new SyncState();
+
+  this.start = new HeaderRecord();
+  this.tip = new HeaderRecord();
+}
+
+/**
+ * Clone the block.
+ * @returns {SyncState}
+ */
+
+SyncState.prototype.clone = function clone() {
+  var state = new SyncState();
+  state.start = this.start.clone();
+  state.tip = this.tip.clone();
+  return state;
+};
+
+/**
+ * Instantiate wallet block from serialized tip data.
+ * @private
+ * @param {Buffer} data
+ */
+
+SyncState.prototype.fromRaw = function fromRaw(data) {
+  var p = new BufferReader(data);
+  this.start.fromRaw(p);
+  this.tip.fromRaw(p);
+  return this;
+};
+
+/**
+ * Instantiate wallet block from serialized data.
+ * @param {Hash} hash
+ * @param {Buffer} data
+ * @returns {SyncState}
+ */
+
+SyncState.fromRaw = function fromRaw(data) {
+  return new SyncState().fromRaw(data);
+};
+
+/**
+ * Serialize the wallet block as a tip (hash and height).
+ * @returns {Buffer}
+ */
+
+SyncState.prototype.toRaw = function toRaw(writer) {
+  var p = new BufferWriter(writer);
+
+  this.start.toRaw(p);
+  this.tip.toRaw(p);
+
+  if (!writer)
+    p = p.render();
+
+  return p;
+};
+
 /**
  * Wallet Tip
  * @constructor
@@ -364,6 +432,7 @@ function serializeWallets(wids) {
  * Expose
  */
 
+exports.SyncState = SyncState;
 exports.HeaderRecord = HeaderRecord;
 exports.BlockMapRecord = BlockMapRecord;
 exports.TXMapRecord = TXMapRecord;

lib/wallet/txdb.js

@@ -1550,13 +1550,13 @@ TXDB.prototype.removeRecursive = co(function* removeRecursive(tx) {
  * @returns {Promise}
  */
 
-TXDB.prototype.unconfirm = co(function* unconfirm(hash, block) {
+TXDB.prototype.unconfirm = co(function* unconfirm(hash) {
   var details;
 
   this.start();
 
   try {
-    details = yield this._unconfirm(hash, block);
+    details = yield this._unconfirm(hash);
   } catch (e) {
     this.drop();
     throw e;
@@ -1574,13 +1574,13 @@ TXDB.prototype.unconfirm = co(function* unconfirm(hash, block) {
  * @returns {Promise}
  */
 
-TXDB.prototype._unconfirm = co(function* unconfirm(hash, block) {
+TXDB.prototype._unconfirm = co(function* unconfirm(hash) {
   var tx = yield this.getTX(hash);
 
   if (!tx)
     return;
 
-  return yield this.disconnect(tx, block);
+  return yield this.disconnect(tx);
 });
 
 /**
@@ -1589,7 +1589,7 @@ TXDB.prototype._unconfirm = co(function* unconfirm(hash, block) {
  * @returns {Promise}
  */
 
-TXDB.prototype.disconnect = co(function* disconnect(tx, block) {
+TXDB.prototype.disconnect = co(function* disconnect(tx) {
   var hash = tx.hash('hex');
   var details = new Details(this, tx);
   var height = tx.height;

lib/wallet/wallet.js

@@ -1965,10 +1965,10 @@ Wallet.prototype._insert = co(function* insert(tx, block) {
  * @returns {Promise}
  */
 
-Wallet.prototype.unconfirm = co(function* unconfirm(hash, block) {
+Wallet.prototype.unconfirm = co(function* unconfirm(hash) {
   var unlock = yield this.writeLock.lock();
   try {
-    return yield this.txdb.unconfirm(hash, block);
+    return yield this.txdb.unconfirm(hash);
   } finally {
     unlock();
   }

lib/wallet/walletdb.js

@@ -24,6 +24,7 @@ var Bloom = require('../utils/bloom');
 var Logger = require('../node/logger');
 var TX = require('../primitives/tx');
 var records = require('./records');
+var SyncState = records.SyncState;
 var BlockMapRecord = records.BlockMapRecord;
 var HeaderRecord = records.HeaderRecord;
 var PathMapRecord = records.PathMapRecord;
@@ -165,10 +166,11 @@ function WalletDB(options) {
   this.logger = options.logger || Logger.global;
   this.client = options.client;
 
-  this.tip = null;
+  this.state = new SyncState();
-  this.height = -1;
   this.depth = 0;
   this.wallets = {};
+  this.genesis = HeaderRecord.fromEntry(this.network.genesis);
+  this.keepBlocks = this.network.block.keepBlocks;
 
   // We need one read lock for `get` and `create`.
   // It will hold locks specific to wallet ids.
@@ -195,12 +197,14 @@ function WalletDB(options) {
     writeBufferSize: 4 << 20,
     bufferKeys: !utils.isBrowser
   });
-
-  this._init();
 }
 
 utils.inherits(WalletDB, AsyncObject);
 
+WalletDB.prototype.__defineGetter__('height', function() {
+  return this.state.tip.height;
+});
+
 /**
  * Database layout.
  * @type {Object}
@@ -208,15 +212,6 @@ utils.inherits(WalletDB, AsyncObject);
 
 WalletDB.layout = layout;
 
-/**
- * Initialize wallet db.
- * @private
- */
-
-WalletDB.prototype._init = function _init() {
-  ;
-};
-
 /**
  * Open the walletdb, wait for the database to load.
  * @alias WalletDB#open
@@ -226,19 +221,20 @@ WalletDB.prototype._init = function _init() {
 WalletDB.prototype._open = co(function* open() {
   yield this.db.open();
   yield this.db.checkVersion('V', 5);
-  yield this.writeGenesis();
+
+  this.depth = yield this.getDepth();
 
   if (this.options.wipeNoReally)
     yield this.wipe();
 
-  this.depth = yield this.getDepth();
+  yield this.init();
+  yield this.watch();
+  yield this.sync();
+  yield this.resend();
 
   this.logger.info(
     'WalletDB loaded (depth=%d, height=%d).',
-    this.depth, this.height);
+    this.depth, this.state.tip.height);
-
-  yield this.connect();
-  yield this.resend();
 });
 
 /**
@@ -260,15 +256,29 @@ WalletDB.prototype._close = co(function* close() {
   yield this.db.close();
 });
 
+/**
+ * Connect and sync with the chain server (without a lock).
+ * @private
+ * @returns {Promise}
+ */
+
+WalletDB.prototype.watch = co(function* watch() {
+  var hashes = yield this.getFilterHashes();
+
+  this.logger.info('Adding %d hashes to filter.', hashes.length);
+
+  this.addFilter(hashes);
+});
+
 /**
  * Connect and sync with the chain server.
  * @returns {Promise}
  */
 
-WalletDB.prototype.connect = co(function* connect() {
+WalletDB.prototype.sync = co(function* sync() {
   var unlock = yield this.txLock.lock();
   try {
-    return yield this._connect();
+    return yield this._sync();
   } finally {
     unlock();
   }
@@ -280,36 +290,36 @@ WalletDB.prototype.connect = co(function* connect() {
  * @returns {Promise}
  */
 
-WalletDB.prototype._connect = co(function* connect() {
+WalletDB.prototype._sync = co(function* connect() {
-  var hashes = yield this.getFilterHashes();
+  var height = this.state.tip.height;
-  var tip, height;
+  var tip, entry;
-
-  this.logger.info('Adding %d hashes to filter.', hashes.length);
-
-  this.addFilter(hashes);
 
   if (!this.client)
     return;
 
-  if (this.options.noScan) {
+  while (height >= 0) {
-    tip = yield this.client.getTip();
+    tip = yield this.getHeader(height);
 
     if (!tip)
-      throw new Error('Could not get chain tip.');
+      break;
 
-    yield this.forceTip(tip);
+    entry = yield this.client.getEntry(tip.hash);
 
-    return;
+    if (entry)
+      break;
+
+    height--;
   }
 
-  assert(this.network.block.keepBlocks > 36);
+  if (!entry) {
+    height = this.state.start.height;
+    entry = yield this.client.getEntry(this.state.start.hash);
 
-  height = this.height - 36;
+    if (!entry)
+      height = 0;
+  }
 
-  if (height < 0)
+  yield this.scan(height);
-    height = 0;
-
-  yield this.scan(height, hashes);
 });
 
 /**
@@ -336,10 +346,7 @@ WalletDB.prototype.rescan = co(function* rescan(height) {
  */
 
 WalletDB.prototype._rescan = co(function* rescan(height) {
-  if (!this.client)
+  return yield this.scan(height);
-    return;
-
-  yield this.scan(height);
 });
 
 /**
@@ -352,26 +359,20 @@ WalletDB.prototype._rescan = co(function* rescan(height) {
 
 WalletDB.prototype.scan = co(function* scan(height) {
   var self = this;
-  var blocks;
 
   if (!this.client)
     return;
 
-  assert(utils.isNumber(height), 'Must pass in a height.');
+  assert(utils.isUInt32(height), 'Must pass in a height.');
 
-  blocks = this.height - height;
+  if (height > this.state.tip.height)
-
-  if (blocks < 0)
     throw new Error('Cannot rescan future blocks.');
 
-  if (blocks > this.network.block.keepBlocks)
-    throw new Error('Cannot roll back beyond keepBlocks.');
-
   yield this.rollback(height);
 
-  this.logger.info('Scanning for blocks.');
+  this.logger.info('Scanning %d blocks.', this.state.tip.height - height);
 
-  yield this.client.scan(this.tip.hash, this.filter, function(block, txs) {
+  yield this.client.scan(this.state.tip.hash, this.filter, function(block, txs) {
     return self._addBlock(block, txs);
   });
 });
@@ -503,7 +504,6 @@ WalletDB.prototype.wipe = co(function* wipe() {
   batch.del(layout.R);
 
   yield batch.write();
-  yield this.writeGenesis();
 });
 
 /**
@@ -542,21 +542,6 @@ WalletDB.prototype.getDepth = co(function* getDepth() {
   return depth + 1;
 });
 
-/**
- * Get current block height.
- * @private
- * @returns {Promise}
- */
-
-WalletDB.prototype.getHeight = co(function* getHeight() {
-  var data = yield this.db.get(layout.R);
-
-  if (!data)
-    return -1;
-
-  return data.readUInt32LE(0, true);
-});
-
 /**
  * Start batch.
  * @private
@@ -1534,26 +1519,24 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) {
  * @returns {Promise}
  */
 
-WalletDB.prototype.writeGenesis = co(function* writeGenesis() {
+WalletDB.prototype.init = co(function* init() {
-  var tip = yield this.getTip();
+  var state = yield this.getState();
+  var tip;
 
-  if (tip) {
+  if (state) {
-    this.tip = tip;
+    this.state = state;
-    this.height = tip.height;
     return;
   }
 
-  yield this.forceTip(this.network.genesis);
+  if (this.client) {
-});
+    tip = yield this.client.getTip();
+    assert(tip);
+    tip = HeaderRecord.fromEntry(tip);
+  } else {
+    tip = this.genesis;
+  }
 
-/**
+  yield this.syncState(tip, true);
- * Write the genesis block as the best hash.
- * @returns {Promise}
- */
-
-WalletDB.prototype.forceTip = co(function* forceTip(entry) {
-  var tip = HeaderRecord.fromEntry(entry);
-  yield this.setTip(tip);
 });
 
 /**
@@ -1561,13 +1544,13 @@ WalletDB.prototype.forceTip = co(function* forceTip(entry) {
  * @returns {Promise}
  */
 
-WalletDB.prototype.getTip = co(function* getTip() {
+WalletDB.prototype.getState = co(function* getState() {
-  var height = yield this.getHeight();
+  var data = yield this.db.get(layout.R);
 
-  if (height === -1)
+  if (!data)
     return;
 
-  return yield this.getHeader(height);
+  return SyncState.fromRaw(data);
 });
 
 /**
@@ -1576,23 +1559,41 @@ WalletDB.prototype.getTip = co(function* getTip() {
  * @returns {Promise}
  */
 
-WalletDB.prototype.setTip = co(function* setTip(tip) {
+WalletDB.prototype.syncState = co(function* syncState(tip, start) {
   var batch = this.db.batch();
-  var height;
+  var state = this.state.clone();
+  var height = this.state.tip.height;
+  var i, blocks;
 
-  batch.del(layout.c(tip.height + 1));
+  if (start)
-  batch.put(layout.c(tip.height), tip.toRaw());
+    state.start = tip;
-  batch.put(layout.R, U32(tip.height));
 
-  height = tip.height - this.network.block.keepBlocks;
+  state.tip = tip;
 
+  // Blocks ahead of our new tip that we need to delete.
+  if (height !== -1) {
+    blocks = height - tip.height;
+    if (blocks > 0) {
+      blocks = Math.min(blocks, this.keepBlocks);
+      for (i = 0; i < blocks; i++) {
+        batch.del(layout.c(height));
+        height--;
+      }
+    }
+  }
+
+  // Prune old blocks.
+  height = tip.height - this.keepBlocks;
   if (height >= 0)
     batch.del(layout.c(height));
 
+  // Save tip and state.
+  batch.put(layout.c(tip.height), tip.toRaw());
+  batch.put(layout.R, state.toRaw());
+
   yield batch.write();
 
-  this.tip = tip;
+  this.state = state;
-  this.height = tip.height;
 });
 
 /**
@@ -1697,24 +1698,34 @@ WalletDB.prototype.getTXMap = co(function* getTXMap(hash) {
  */
 
 WalletDB.prototype.rollback = co(function* rollback(height) {
-  var tip;
+  var tip, start;
 
-  if (this.height > height) {
+  if (this.state.tip.height <= height)
-    this.logger.info(
+    return;
-      'Rolling back %d blocks to height %d.',
-      this.height - height, height);
-  }
 
-  while (this.height > height) {
+  this.logger.info(
-    tip = yield this.getHeader(this.height);
+    'Rolling back %d blocks to height %d.',
+    this.state.tip.height - height, height);
 
-    if (!tip) {
+  tip = yield this.getHeader(height);
-      yield this.forceTip(this.network.genesis);
+
-      throw new Error('Wallet reorgd beyond safe height. Rescan required.');
+  if (!tip) {
+    if (height >= this.state.start.height) {
+      yield this.revert(this.state.start.height);
+      yield this.syncState(this.state.start, true);
+      this.logger.warning(
+        'Wallet rolled back to start block (%d).',
+        this.state.tip.height);
+    } else {
+      yield this.revert(0);
+      yield this.syncState(this.genesis, true);
+      this.logger.warning('Wallet rolled back to genesis block.');
     }
-
+    return;
-    yield this._removeBlock(tip);
   }
+
+  yield this.revert(height);
+  yield this.syncState(tip);
 });
 
 /**
@@ -1724,6 +1735,7 @@ WalletDB.prototype.rollback = co(function* rollback(height) {
  */
 
 WalletDB.prototype.revert = co(function* revert(height) {
+  var total = 0;
   var i, iter, item, block, tx;
 
   iter = this.db.iterator({
@@ -1740,6 +1752,7 @@ WalletDB.prototype.revert = co(function* revert(height) {
 
     try {
       block = BlockMapRecord.fromRaw(item.value);
+      total += block.txs.length;
 
       for (i = block.txs.length - 1; i >= 0; i--) {
         tx = block.txs[i];
@@ -1750,6 +1763,8 @@ WalletDB.prototype.revert = co(function* revert(height) {
       throw e;
     }
   }
+
+  this.logger.info('Rolled back %d transactions.', total);
 });
 
 /**
@@ -1778,17 +1793,20 @@ WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) {
   var total = 0;
   var i, tip, tx;
 
-  if (entry.height <= this.height) {
+  if (entry.height < this.state.tip.height) {
     this.logger.warning('Wallet is connecting low blocks.');
     return total;
   }
 
-  if (entry.height !== this.height + 1)
+  if (entry.height === this.state.tip.height) {
+    this.logger.warning('Wallet is connecting low blocks.');
+  } else if (entry.height !== this.state.tip.height + 1) {
     throw new Error('Bad connection (height mismatch).');
+  }
 
   tip = HeaderRecord.fromEntry(entry);
 
-  yield this.setTip(tip);
+  yield this.syncState(tip);
 
   if (this.options.useCheckpoints) {
     if (tip.height <= this.network.checkpoints.lastHeight)
@@ -1835,12 +1853,12 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) {
 WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
   var i, tx, tip, prev, block;
 
-  if (entry.height > this.height) {
+  if (entry.height > this.state.tip.height) {
     this.logger.warning('Wallet is disconnecting high blocks.');
     return 0;
   }
 
-  if (entry.height !== this.height)
+  if (entry.height !== this.state.tip.height)
     throw new Error('Bad disconnection (height mismatch).');
 
   tip = yield this.getHeader(entry.height);
@@ -1856,7 +1874,7 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
   block = yield this.getBlockMap(tip.height);
 
   if (!block) {
-    yield this.setTip(prev);
+    yield this.syncState(prev);
     return 0;
   }
 
@@ -1865,7 +1883,7 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
     yield this._unconfirm(tx, tip);
   }
 
-  yield this.setTip(prev);
+  yield this.syncState(prev);
 
   this.logger.warning('Disconnected block %s (tx=%d).',
     utils.revHex(tip.hash), block.txs.length);
@@ -1956,14 +1974,14 @@ WalletDB.prototype._insert = co(function* insert(tx, block) {
  * @returns {Promise}
  */
 
-WalletDB.prototype._unconfirm = co(function* unconfirm(tx, block) {
+WalletDB.prototype._unconfirm = co(function* unconfirm(tx) {
   var i, wid, wallet;
 
   for (i = 0; i < tx.wids.length; i++) {
     wid = tx.wids[i];
     wallet = yield this.get(wid);
     assert(wallet);
-    yield wallet.unconfirm(tx.hash, block);
+    yield wallet.unconfirm(tx.hash);
   }
 });
 

test/chain-test.js

@@ -228,7 +228,7 @@ describe('Chain', function() {
     assert(wallet.account.changeDepth >= 7);
 
     assert.equal(walletdb.height, chain.height);
-    assert.equal(walletdb.tip.hash, chain.tip.hash);
+    assert.equal(walletdb.state.tip.hash, chain.tip.hash);
 
     txs = yield wallet.getHistory();
     assert.equal(txs.length, 44);