wallet: more chain atomicity.

lib/http/server.js

@@ -1574,25 +1574,6 @@ ClientSocket.prototype.testFilter = function testFilter(tx) {
 
 ClientSocket.prototype.scan = co(function* scan(start) {
   var scanner = this.scanner.bind(this);
-  var entry;
-
-  if (this.chain.db.options.spv) {
-    entry = yield this.chain.db.get(start);
-
-    if (!entry)
-      throw new Error('Block not found.');
-
-    if (!entry.isGenesis())
-      start = entry.prevBlock;
-
-    yield this.chain.reset(start);
-
-    return;
-  }
-
-  if (this.chain.db.options.prune)
-    throw new Error('Cannot scan in pruned mode.');
-
   yield this.chain.db.scan(start, this.filter, scanner);
 });
 

lib/node/config.js

@@ -162,7 +162,6 @@ config.parseData = function parseData(data, prefix, dirname) {
   options.coinCache = bool(data.coincache);
   options.indexTX = bool(data.indextx);
   options.indexAddress = bool(data.indexaddress);
-  options.noScan = bool(data.noscan);
 
   // Mempool
   options.limitFree = bool(data.limitfree);
@@ -191,7 +190,6 @@ config.parseData = function parseData(data, prefix, dirname) {
   // Miner
   options.payoutAddress = str(data.payoutaddress);
   options.coinbaseFlags = str(data.coinbaseflags);
-  options.parallel = bool(data.parallel);
 
   // HTTP
   options.sslCert = file(data.sslcert, prefix, dirname);
@@ -203,6 +201,9 @@ config.parseData = function parseData(data, prefix, dirname) {
   options.walletAuth = bool(data.walletauth);
   options.noAuth = bool(data.noauth);
 
+  // Wallet
+  options.wipeNoReally = bool(data.wipenoreally);
+
   options.data = data;
 
   if (options.knownPeers != null)

lib/node/fullnode.js

@@ -148,6 +148,7 @@ function Fullnode(options) {
     witness: this.options.witness,
     useCheckpoints: this.options.useCheckpoints,
     maxFiles: this.options.maxFiles,
+    wipeNoReally: this.options.wipeNoReally,
     resolution: false,
     verify: false
   });
@@ -288,12 +289,6 @@ Fullnode.prototype._close = co(function* close() {
  */
 
 Fullnode.prototype.rescan = function rescan() {
-  if (this.options.noScan) {
-    return this.walletdb.setTip(
-      this.chain.tip.hash,
-      this.chain.height);
-  }
-
   // Always rescan to make sure we didn't
   // miss anything: there is no atomicity
   // between the chaindb and walletdb.

lib/node/spvnode.js

@@ -86,6 +86,7 @@ function SPVNode(options) {
     location: this.location('walletdb'),
     witness: this.options.witness,
     maxFiles: this.options.maxFiles,
+    wipeNoReally: this.options.wipeNoReally,
     resolution: true,
     verify: true
   });
@@ -137,11 +138,18 @@ SPVNode.prototype._init = function _init() {
     self.walletdb.addTX(tx).catch(onError);
   });
 
-  this.chain.on('block', function(block, entry) {
+  this.chain.on('block', function(block) {
     self.emit('block', block);
+  });
+
+  this.chain.on('connect', function(entry, block) {
     self.walletdb.addBlock(entry, block.txs).catch(onError);
   });
 
+  this.chain.on('disconnect', function(entry, block) {
+    self.walletdb.removeBlock(entry).catch(onError);
+  });
+
   this.walletdb.on('path', function(path) {
     self.pool.watch(path.hash, 'hex');
   });
@@ -223,12 +231,6 @@ SPVNode.prototype.openFilter = co(function* openFilter() {
  */
 
 SPVNode.prototype.rescan = function rescan() {
-  if (this.options.noScan) {
-    return this.walletdb.setTip(
-      this.chain.tip.hash,
-      this.chain.height);
-  }
-
   // Always replay the last block to make
   // sure we didn't miss anything: there
   // is no atomicity between the chaindb
@@ -244,19 +246,7 @@ SPVNode.prototype.rescan = function rescan() {
  */
 
 SPVNode.prototype.scan = co(function* scan(height) {
-  if (height == null)
+  return this.walletdb.rescan(this.chain.db, height);
-    height = this.walletdb.height;
-
-  if (typeof height === 'string') {
-    height = yield this.chain.db.getHeight(height);
-    if (height === -1)
-      return;
-  }
-
-  if (height === 0)
-    return;
-
-  yield this.chain.reset(height - 1);
 });
 
 /**

lib/wallet/txdb.js

@@ -862,6 +862,81 @@ TXDB.prototype.isSpent = co(function* isSpent(hash, index) {
   return data != null;
 });
 
+TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) {
+  var hash = tx.hash('hex');
+  var wids = yield this.walletdb.getWalletsByTX(hash);
+  var result;
+
+  if (!wids)
+    wids = [];
+
+  result = utils.binaryInsert(wids, this.wallet.wid, cmp, true);
+
+  if (result === -1)
+    return;
+
+  this.walletdb.writeTX(this.wallet, hash, wids);
+});
+
+TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) {
+  var hash = tx.hash('hex');
+  var wids = yield this.walletdb.getWalletsByTX(hash);
+  var result, block;
+
+  if (!wids)
+    return;
+
+  result = utils.binaryRemove(wids, this.wallet.wid, cmp);
+
+  if (!result)
+    return;
+
+  if (tx.height !== -1) {
+    block = yield this.walletdb.getBlock(tx.height);
+
+    if (block.remove(hash, this.wallet.wid))
+      this.walletdb.writeBlock(this.wallet, block);
+  }
+
+  if (wids.length === 0) {
+    this.walletdb.unwriteTX(this.wallet, hash);
+    return;
+  }
+
+  this.walletdb.writeTX(this.wallet, hash, wids);
+});
+
+TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, record) {
+  var hash = tx.hash('hex');
+  var block = yield this.walletdb.getBlock(record.height);
+
+  if (!block)
+    block = record.clone();
+
+  if (!block.add(hash, this.wallet.wid))
+    return;
+
+  this.walletdb.writeBlock(this.wallet, block);
+});
+
+TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) {
+  var hash = tx.hash('hex');
+  var block = yield this.walletdb.getBlock(height);
+
+  if (!block)
+    return;
+
+  if (!block.remove(hash, this.wallet.wid))
+    return;
+
+  if (block.txs.length === 0) {
+    this.walletdb.unwriteBlock(this.wallet, block);
+    return;
+  }
+
+  this.walletdb.writeBlock(this.wallet, block);
+});
+
 /**
  * Add transaction, potentially runs
  * `confirm()` and `removeConflicts()`.
@@ -901,23 +976,19 @@ TXDB.prototype._add = co(function* add(tx, block) {
 
   if (existing) {
     // Existing tx is already confirmed. Ignore.
-    if (existing.height !== -1) {
+    if (existing.height !== -1)
-      if (block) {
-        // XXX need to return true here, could be a rescan.
-      }
       return;
-    }
 
     // The incoming tx won't confirm the
     // existing one anyway. Ignore.
     if (tx.height === -1)
       return;
 
-    // Save the original time.
+    // Save the index (can't get this elsewhere).
-    tx.ps = existing.ps;
+    existing.index = tx.index;
 
     // Confirm transaction.
-    return yield this._confirm(tx, block);
+    return yield this._confirm(existing, block);
   }
 
   if (tx.height === -1) {
@@ -1118,81 +1189,6 @@ TXDB.prototype.insert = co(function* insert(tx, block) {
   return details;
 });
 
-TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) {
-  var hash = tx.hash('hex');
-  var wids = yield this.walletdb.getWalletsByTX(hash);
-  var result;
-
-  if (!wids)
-    wids = [];
-
-  result = utils.binaryInsert(wids, this.wallet.wid, cmp, true);
-
-  if (result === -1)
-    return;
-
-  this.walletdb.writeTX(this.wallet, hash, wids);
-});
-
-TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) {
-  var hash = tx.hash('hex');
-  var wids = yield this.walletdb.getWalletsByTX(hash);
-  var result;
-
-  if (!wids)
-    return;
-
-  result = utils.binaryRemove(wids, this.wallet.wid, cmp);
-
-  if (!result)
-    return;
-
-  yield this.removeBlockRecord(tx, tx.height);
-
-  if (wids.length === 0) {
-    this.walletdb.unwriteTX(this.wallet, hash);
-    return;
-  }
-
-  this.walletdb.writeTX(this.wallet, hash, wids);
-});
-
-TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, record) {
-  var hash = tx.hash('hex');
-  var block = yield this.walletdb.getBlock(record.height);
-
-  if (!block)
-    block = record.clone();
-
-  if (!block.add(hash, this.wallet.wid))
-    return;
-
-  this.walletdb.writeBlock(this.wallet, block);
-});
-
-TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) {
-  var hash = tx.hash('hex');
-  var block = yield this.walletdb.getBlock(height);
-
-  if (!block)
-    return;
-
-  if (!block.remove(hash, this.wallet.wid))
-    return;
-
-  if (block.txs.length === 0) {
-    this.walletdb.unwriteBlock(this.wallet, block);
-    return;
-  }
-
-  this.walletdb.writeBlock(this.wallet, block);
-});
-
-
-function cmp(a, b) {
-  return a - b;
-}
-
 /**
  * Attempt to confirm a transaction.
  * @private
@@ -1207,11 +1203,6 @@ TXDB.prototype.confirm = co(function* confirm(hash, block) {
   if (!tx)
     return;
 
-  tx.height = block.height;
-  tx.block = block.hash;
-  tx.index = -1;
-  tx.ts = utils.now();
-
   this.start();
 
   try {
@@ -1240,7 +1231,12 @@ TXDB.prototype._confirm = co(function* confirm(tx, block) {
   var i, account, output, coin, input, prevout;
   var path, credit, credits;
 
-  assert(tx.height !== -1);
+  if (!block)
+    block = this.wallet.db.tip;
+
+  tx.height = block.height;
+  tx.block = block.hash;
+  tx.ts = block.ts;
 
   if (!tx.isCoinbase()) {
     credits = yield this.getSpentCredits(tx);

lib/wallet/walletdb.js

@@ -152,8 +152,8 @@ function WalletDB(options) {
   this.fees = options.fees;
   this.logger = options.logger || Logger.global;
 
-  this.tip = this.network.genesis.hash;
+  this.tip = null;
-  this.height = 0;
+  this.height = -1;
   this.depth = 0;
   this.wallets = {};
 
@@ -217,6 +217,9 @@ WalletDB.prototype._open = co(function* open() {
   yield this.db.checkVersion('V', 4);
   yield this.writeGenesis();
 
+  if (this.options.wipeNoReally)
+    yield this.wipe();
+
   this.depth = yield this.getDepth();
 
   this.logger.info(
@@ -255,6 +258,55 @@ WalletDB.prototype.backup = function backup(path) {
   return this.db.backup(path);
 };
 
+/**
+ * Wipe the txdb - NEVER USE.
+ * @returns {Promise}
+ */
+
+WalletDB.prototype.wipe = co(function* wipe() {
+  var batch = this.db.batch();
+  var dummy = new Buffer(0);
+  var i, keys, key;
+
+  this.logger.warning('Wiping txdb...');
+  this.logger.warning('I hope you know what you\'re doing.');
+
+  keys = yield this.db.keys({
+    gte: TXDB.layout.prefix(0x00000000, dummy),
+    lte: TXDB.layout.prefix(0xffffffff, dummy)
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+
+  keys = yield this.db.keys({
+    gte: layout.b(constants.NULL_HASH),
+    lte: layout.b(constants.HIGH_HASH)
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+
+  keys = yield this.db.keys({
+    gte: layout.e(constants.NULL_HASH),
+    lte: layout.e(constants.HIGH_HASH)
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+
+  batch.del(layout.R);
+
+  yield batch.write();
+  yield this.writeGenesis();
+});
+
 /**
  * Get current wallet wid depth.
  * @private
@@ -1104,7 +1156,7 @@ WalletDB.prototype._rescan = co(function* rescan(chaindb, height) {
 
   this.logger.info('Scanning for %d addresses.', hashes.length);
 
-  yield chaindb.scan(this.tip, hashes, function(block, txs) {
+  yield chaindb.scan(this.tip.hash, hashes, function(block, txs) {
     return self._addBlock(block, txs);
   });
 });
@@ -1317,12 +1369,15 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) {
 
 WalletDB.prototype.writeGenesis = co(function* writeGenesis() {
   var block = yield this.getTip();
+  var genesis = this.network.genesis;
+
   if (block) {
-    this.tip = block.hash;
+    this.tip = block;
     this.height = block.height;
     return;
   }
-  yield this.setTip(this.network.genesis.hash, 0);
+
+  yield this.setTip(genesis.hash, 0, genesis.ts);
 });
 
 /**
@@ -1347,8 +1402,8 @@ WalletDB.prototype.getTip = co(function* getTip() {
  * @returns {Promise}
  */
 
-WalletDB.prototype.setTip = function setTip(hash, height) {
+WalletDB.prototype.setTip = function setTip(hash, height, ts) {
-  var block = new WalletBlock(hash, height);
+  var block = new WalletBlock(hash, height, ts);
   return this.setBlock(block);
 };
 
@@ -1368,7 +1423,7 @@ WalletDB.prototype.setBlock = co(function* setBlock(block) {
 
   yield batch.write();
 
-  this.tip = block.hash;
+  this.tip = block;
   this.height = block.height;
 });
 
@@ -1481,6 +1536,15 @@ WalletDB.prototype.addBlock = co(function* addBlock(entry, txs) {
   var unlock = yield this.txLock.lock();
   try {
     yield this.rollback(entry.height - 1);
+
+    if (entry.height <= this.height) {
+      this.logger.warning('Node is connecting low blocks in wallet.');
+      return;
+    }
+
+    if (entry.height !== this.height + 1)
+      throw new Error('Bad connection (height mismatch).');
+
     return yield this._addBlock(entry, txs);
   } finally {
     unlock();
@@ -1523,7 +1587,7 @@ WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) {
   }
 
   this.height = block.height;
-  this.tip = block.hash;
+  this.tip = block;
 
   this.logger.info('Connected block %s (tx=%d).',
     utils.revHex(block.hash), total);
@@ -1542,6 +1606,15 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) {
   var unlock = yield this.txLock.lock();
   try {
     yield this.rollback(entry.height);
+
+    if (entry.height > this.height) {
+      this.logger.warning('Node is disconnecting high blocks in wallet.');
+      return;
+    }
+
+    if (entry.height !== this.height)
+      throw new Error('Bad disconnection (height mismatch).');
+
     return yield this._removeBlock(entry);
   } finally {
     unlock();
@@ -1562,8 +1635,6 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
   if (!block)
     return;
 
-  assert(block.height > 0);
-
   prev = yield this.getBlock(entry.height - 1);
   assert(prev);
 
@@ -1578,7 +1649,7 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) {
   }
 
   this.height = prev.height;
-  this.tip = prev.hash;
+  this.tip = prev;
 
   this.logger.warning('Disconnected block %s (tx=%d).',
     utils.revHex(block.hash), block.txs.length);
@@ -1645,25 +1716,6 @@ WalletDB.prototype._insertTX = co(function* insertTX(tx, block) {
   return wids;
 });
 
-/**
- * Confirm a transaction from all
- * relevant wallets without a lock.
- * @private
- * @param {TXHash} hash
- * @returns {Promise}
- */
-
-WalletDB.prototype._confirmTX = co(function* confirmTX(tx, block) {
-  var i, wid, wallet;
-
-  for (i = 0; i < tx.wids.length; i++) {
-    wid = tx.wids[i];
-    wallet = yield this.get(wid);
-    assert(wallet);
-    yield wallet.confirm(tx.hash, block);
-  }
-});
-
 /**
  * Unconfirm a transaction from all
  * relevant wallets without a lock.
@@ -1724,12 +1776,13 @@ WalletDB.prototype._zap = co(function* zap(age) {
  * @param {Number} height
  */
 
-function WalletBlock(hash, height) {
+function WalletBlock(hash, height, ts) {
   if (!(this instanceof WalletBlock))
-    return new WalletBlock(hash, height);
+    return new WalletBlock(hash, height, ts);
 
   this.hash = hash || constants.NULL_HASH;
   this.height = height != null ? height : -1;
+  this.ts = ts || 0;
   this.txs = [];
   this.index = {};
 }
@@ -1740,7 +1793,7 @@ function WalletBlock(hash, height) {
  */
 
 WalletBlock.prototype.clone = function clone() {
-  return new WalletBlock(this.hash, this.height);
+  return new WalletBlock(this.hash, this.height, this.ts);
 };
 
 /**
@@ -1752,6 +1805,7 @@ WalletBlock.prototype.clone = function clone() {
 WalletBlock.prototype.fromEntry = function fromEntry(entry) {
   this.hash = entry.hash;
   this.height = entry.height;
+  this.ts = entry.ts;
   return this;
 };
 
@@ -1764,6 +1818,7 @@ WalletBlock.prototype.fromEntry = function fromEntry(entry) {
 WalletBlock.prototype.fromJSON = function fromJSON(json) {
   this.hash = utils.revHex(json.hash);
   this.height = json.height;
+  this.ts = json.ts;
   return this;
 };
 
@@ -1780,6 +1835,7 @@ WalletBlock.prototype.fromRaw = function fromRaw(data) {
 
   this.hash = p.readHash('hex');
   this.height = p.readU32();
+  this.ts = p.readU32();
 
   while (p.left()) {
     hash = p.readHash('hex');
@@ -1804,6 +1860,7 @@ WalletBlock.prototype.fromTip = function fromTip(data) {
   var p = new BufferReader(data);
   this.hash = p.readHash('hex');
   this.height = p.readU32();
+  this.ts = p.readU32();
   return this;
 };
 
@@ -1858,6 +1915,7 @@ WalletBlock.prototype.toTip = function toTip(writer) {
 
   p.writeHash(this.hash);
   p.writeU32(this.height);
+  p.writeU32(this.ts);
 
   if (!writer)
     p = p.render();
@@ -1877,6 +1935,7 @@ WalletBlock.prototype.toRaw = function toRaw(writer) {
 
   p.writeHash(this.hash);
   p.writeU32(this.height);
+  p.writeU32(this.ts);
 
   for (i = 0; i < this.txs.length; i++) {
     tx = this.txs[i];
@@ -1901,7 +1960,6 @@ WalletBlock.prototype.toRaw = function toRaw(writer) {
 
 WalletBlock.prototype.add = function add(hash, wid) {
   var tx = this.index[hash];
-  var index;
 
   if (!tx) {
     tx = new TXHash(hash);
@@ -1911,12 +1969,7 @@ WalletBlock.prototype.add = function add(hash, wid) {
     return true;
   }
 
-  index = utils.binaryInsert(tx.wids, wid, cmp, true);
+  return tx.add(wid);
-
-  if (index === -1)
-    return false;
-
-  return true;
 };
 
 /**
@@ -1933,9 +1986,7 @@ WalletBlock.prototype.remove = function remove(hash, wid) {
   if (!tx)
     return false;
 
-  result = utils.binaryRemove(tx.wids, wid, cmp);
+  if (!tx.remove(wid))
-
-  if (!result)
     return false;
 
   if (tx.wids.length === 0) {
@@ -1971,6 +2022,26 @@ function TXHash(hash, wids) {
 
 TXHash.id = 0;
 
+TXHash.prototype.add = function add(wid) {
+  return utils.binaryInsert(this.wids, wid, cmp, true) !== -1;
+};
+
+TXHash.prototype.remove = function remove(wid) {
+  return utils.binaryRemove(this.wids, wid, cmp);
+};
+
+TXHash.prototype.toRaw = function toRaw(writer) {
+  return serializeWallets(this.wids, writer);
+};
+
+TXHash.prototype.fromRaw = function fromRaw(data) {
+  return parseWallets(data);
+};
+
+TXHash.fromRaw = function fromRaw(hash, data) {
+  return new TXHash(hash).fromRaw(data);
+};
+
 function parseWallets(data) {
   var p = new BufferReader(data);
   var wids = [];
@@ -1981,8 +2052,8 @@ function parseWallets(data) {
   return wids;
 }
 
-function serializeWallets(wids) {
+function serializeWallets(wids, writer) {
-  var p = new BufferWriter();
+  var p = new BufferWriter(writer);
   var i, wid;
 
   for (i = 0; i < wids.length; i++) {
@@ -1990,7 +2061,10 @@ function serializeWallets(wids) {
     p.writeU32(wid);
   }
 
-  return p.render();
+  if (!writer)
+    p = p.render();
+
+  return p;
 }
 
 function cmp(a, b) {

test/chain-test.js

@@ -227,7 +227,7 @@ describe('Chain', function() {
     assert(wallet.account.changeDepth >= 7);
 
     assert.equal(walletdb.height, chain.height);
-    assert.equal(walletdb.tip, chain.tip.hash);
+    assert.equal(walletdb.tip.hash, chain.tip.hash);
 
     txs = yield wallet.getHistory();
     assert.equal(txs.length, 44);