walletdb: better block management.

lib/bcoin/chaindb.js

@@ -1138,7 +1138,7 @@ ChainDB.prototype.scan = function scan(start, filter, iter, callback) {
     if (err)
       return callback(err);
 
-    if (!hash) {
+    if (hash == null) {
       self.logger.info('Finished scanning %d blocks.', total);
       return callback();
     }
@@ -1181,7 +1181,7 @@ ChainDB.prototype.scan = function scan(start, filter, iter, callback) {
         if (txs.length === 0)
           return self.getNextHash(hash, next);
 
-        iter(txs, entry, function(err) {
+        iter(entry, txs, function(err) {
           if (err)
             return next(err);
           self.getNextHash(hash, next);

lib/bcoin/fullnode.js

@@ -298,7 +298,7 @@ Fullnode.prototype._open = function open(callback) {
     },
     function(next) {
       if (self.options.noScan) {
-        self.walletdb.setTip(self.chain.tip.hash, 0, next);
+        self.walletdb.setTip(self.chain.tip.hash, self.chain.height, next);
         return next();
       }
       // Always rescan to make sure we didn't miss anything:

lib/bcoin/http/server.js

@@ -1294,7 +1294,7 @@ ClientSocket.prototype.watchChain = function watchChain() {
     txs = self.testBlock(block);
 
     if (txs)
-      self.emit('block tx', txs, entry.toJSON());
+      self.emit('block tx', entry.toJSON(), txs);
   });
 
   this.bind(this.chain, 'disconnect', function(entry, block) {
@@ -1353,11 +1353,11 @@ ClientSocket.prototype.scan = function scan(start, callback) {
 
   start = utils.revHex(start);
 
-  this.chain.db.scan(start, this.filter, function(txs, entry, next) {
+  this.chain.db.scan(start, this.filter, function(entry, txs, next) {
     for (i = 0; i < txs.length; i++)
       txs[i] = txs[i].toJSON();
 
-    self.emit('block tx', txs, entry.toJSON());
+    self.emit('block tx', entry.toJSON(), txs);
     next();
   }, function(err) {
     if (err)

lib/bcoin/txdb.js

@@ -11,16 +11,17 @@
  * Database Layout:
  *   t/[hash] -> extended tx
  *   c/[hash]/[index] -> coin
+ *   d/[hash]/[index] -> undo coin
  *   s/[hash]/[index] -> spent by hash
  *   o/[hash]/[index] -> orphan inputs
  *   p/[hash] -> dummy (pending flag)
  *   m/[time]/[hash] -> dummy (tx by time)
  *   h/[height]/[hash] -> dummy (tx by height)
- *   T/[id]/[name]/[hash] -> dummy (tx by wallet id)
+ *   T/[account]/[hash] -> dummy (tx by account)
- *   P/[id]/[name]/[hash] -> dummy (pending tx by wallet/account id)
+ *   P/[account]/[hash] -> dummy (pending tx by account)
- *   M/[id]/[name]/[time]/[hash] -> dummy (tx by time + id/account)
+ *   M/[account]/[time]/[hash] -> dummy (tx by time + account)
- *   H/[id]/[name]/[height]/[hash] -> dummy (tx by height + id/account)
+ *   H/[account]/[height]/[hash] -> dummy (tx by height + account)
- *   C/[id]/[name]/[hash]/[index] -> dummy (coin by id/account)
+ *   C/[account]/[hash]/[index] -> dummy (coin by account)
  */
 
 var bcoin = require('./env');

lib/bcoin/utils.js

@@ -2411,8 +2411,8 @@ utils.isAlpha = function isAlpha(key) {
   if (typeof key !== 'string')
     return false;
   // We allow /-~ (exclusive), 0-} (inclusive)
-  return key.length > 0
+  return key.length >= 1
-    && key.length <= 64
+    && key.length <= 40
     && /^[\u0030-\u007d]+$/.test(key);
 };
 

lib/bcoin/wallet.js

@@ -413,25 +413,31 @@ Wallet.prototype.unlock = function unlock(passphrase, timeout, callback) {
 
 /**
  * Generate the wallet ID if none was passed in.
- * It is represented as `m/44` (public) hashed
+ * It is represented as HASH160(m/44->public|magic)
- * and converted to an address with a prefix
+ * converted to an "address" with a prefix
  * of `0x03be04` (`WLT` in base58).
  * @private
  * @returns {Base58String}
  */
 
 Wallet.prototype.getID = function getID() {
-  var key, p;
+  var key, p, hash;
 
   assert(this.master.key, 'Cannot derive id.');
 
   key = this.master.key.derive(44);
 
+  p = new BufferWriter();
+  p.writeBytes(key.publicKey);
+  p.writeU32(this.network.magic);
+
+  hash = utils.hash160(p.render());
+
   p = new BufferWriter();
   p.writeU8(0x03);
   p.writeU8(0xbe);
   p.writeU8(0x04);
-  p.writeBytes(utils.hash160(key.publicKey));
+  p.writeBytes(hash);
   p.writeChecksum();
 
   return utils.toBase58(p.render());

lib/bcoin/walletdb.js

@@ -14,6 +14,9 @@
  *  w/[id] -> wallet
  *  a/[id]/[index] -> account
  *  i/[id]/[name] -> account index
+ *  R -> tip
+ *  b/[hash] -> wallet block
+ *  e/[hash] -> tx->wallet-id map
  */
 
 var bcoin = require('./env');
@@ -928,7 +931,7 @@ WalletDB.prototype.rescan = function rescan(chaindb, callback) {
 
     self.logger.info('Scanning for %d addresses.', hashes.length);
 
-    chaindb.scan(self.tip, hashes, function(txs, block, next) {
+    chaindb.scan(self.height, hashes, function(block, txs, next) {
       self.addBlock(block, txs, next);
     }, callback);
   });
@@ -1064,17 +1067,17 @@ WalletDB.prototype.getTable = function getTable(addresses, callback) {
 WalletDB.prototype.writeGenesis = function writeGenesis(callback) {
   var self = this;
 
-  this.getTip(function(err, hash, height) {
+  this.getTip(function(err, block) {
     if (err)
       return callback(err);
 
-    if (hash) {
+    if (block) {
-      self.tip = hash;
+      self.tip = block.hash;
-      self.height = height;
+      self.height = block.height;
       return callback();
     }
 
-    self.setTip(self.tip, self.height, callback);
+    self.setTip(self.network.genesis.hash, 0, callback);
   });
 };
 
@@ -1085,54 +1088,121 @@ WalletDB.prototype.writeGenesis = function writeGenesis(callback) {
 
 WalletDB.prototype.getTip = function getTip(callback) {
   this.db.fetch('R', function(data) {
-    var p = new BufferReader(data);
+    return WalletBlock.fromTip(data);
-    return [p.readHash('hex'), p.readU32()];
+  }, callback);
-  }, function(err, items) {
-    if (err)
-      return callback(err);
-
-    if (!items)
-      return callback(null, null, -1);
-
-    return callback(null, items[0], items[1]);
-  });
 };
 
 /**
  * Write the best block hash.
  * @param {Hash} hash
+ * @param {Number} height
  * @param {Function} callback
  */
 
 WalletDB.prototype.setTip = function setTip(hash, height, callback) {
   var self = this;
-  var p = new BufferWriter();
+  var block = new WalletBlock(hash, height);
-
+  this.db.put('R', block.toTip(), function(err) {
-  p.writeHash(hash);
-  p.writeU32(height);
-
-  this.db.put('R', p.render(), function(err) {
     if (err)
       return callback(err);
 
-    self.tip = hash;
+    self.tip = block.hash;
-    self.height = height;
+    self.height = block.height;
 
     return callback();
   });
 };
 
 /**
- * Add a block's transactions and write the new best hash.
+ * Connect a block.
- * @param {Block} block
+ * @param {WalletBlock} block
  * @param {Function} callback
  */
 
-WalletDB.prototype.addBlock = function addBlock(block, txs, callback, force) {
+WalletDB.prototype.writeBlock = function writeBlock(block, matches, callback) {
-  var self = this;
+  var batch = this.db.batch();
-  var unlock;
+  var i, hash, wallets;
 
-  unlock = this.locker.lock(addBlock, [block, txs, callback], force);
+  batch.put('R', block.toTip());
+
+  if (block.hashes.length > 0) {
+    batch.put('b/' + block.hash, block.toRaw());
+
+    for (i = 0; i < block.hashes.length; i++) {
+      hash = block.hashes[i];
+      wallets = matches[i];
+      batch.put('e/' + hash, serializeWallets(wallets));
+    }
+  }
+
+  batch.write(function(err) {
+    if (err)
+      return callback(err);
+
+    self.tip = block.hash;
+    self.height = block.height;
+
+    return callback();
+  });
+};
+
+/**
+ * Disconnect a block.
+ * @param {WalletBlock} block
+ * @param {Function} callback
+ */
+
+WalletDB.prototype.unwriteBlock = function unwriteBlock(block, callback) {
+  var batch = this.db.batch();
+  var prev = new WalletBlock(block.prevBlock, block.height - 1);
+
+  batch.put('R', prev.toTip());
+  batch.del('b/' + block.hash);
+
+  batch.write(function(err) {
+    if (err)
+      return callback(err);
+
+    self.tip = prev.hash;
+    self.height = prev.height;
+
+    return callback();
+  });
+};
+
+/**
+ * Get a wallet block (with hashes).
+ * @param {Hash} hash
+ * @param {Function} callback
+ */
+
+WalletDB.prototype.getBlock = function getBlock(hash, callback) {
+  this.db.fetch('b/' + hash, function(err, data) {
+    return WalletBlock.fromRaw(hash, data);
+  }, callback);
+};
+
+/**
+ * Get a TX->Wallet map.
+ * @param {Hash} hash
+ * @param {Function} callback
+ */
+
+WalletDB.prototype.getWalletsByTX = function getWalletsByTX(hash, callback) {
+  this.db.fetch('e/' + hash, parseWallets, callback);
+};
+
+/**
+ * Add a block's transactions and write the new best hash.
+ * @param {ChainEntry} entry
+ * @param {Function} callback
+ */
+
+WalletDB.prototype.addBlock = function addBlock(entry, txs, callback, force) {
+  var self = this;
+  var i, block, matches, hash, unlock;
+
+  unlock = this.locker.lock(addBlock, [entry, txs, callback], force);
 
   if (!unlock)
     return;
@@ -1140,66 +1210,98 @@ WalletDB.prototype.addBlock = function addBlock(block, txs, callback, force) {
   callback = utils.wrap(callback, unlock);
 
   if (this.options.useCheckpoints) {
-    if (block.height < this.network.checkpoints.lastHeight)
+    if (entry.height <= this.network.checkpoints.lastHeight)
-      return this.setTip(block.hash, block.height, callback);
+      return this.setTip(entry.hash, entry.height, callback);
   }
 
-  if (!Array.isArray(txs))
+  block = WalletBlock.fromEntry(entry);
-    txs = [txs];
+  matches = [];
 
+  // NOTE: Atomicity doesn't matter here. If we crash
+  // during this loop, the automatic rescan will get
+  // the database back into the correct state.
   utils.forEachSerial(txs, function(tx, next) {
-    self.addTX(tx, next, true);
+    self.addTX(tx, function(err, wallets) {
+      if (err)
+        return next(err);
+
+      if (!wallets)
+        return next();
+
+      hash = tx.hash('hex');
+      block.hashes.push(hash);
+      matches.push(wallets);
+
+      next();
+    }, true);
   }, function(err) {
     if (err)
       return callback(err);
 
-    self.setTip(block.hash, block.height, callback);
+    self.writeBlock(block, matches, callback);
   });
 };
 
 /**
  * Unconfirm a block's transactions
  * and write the new best hash (SPV version).
- * @param {Block} block
+ * @param {ChainEntry} entry
  * @param {Function} callback
  */
 
-WalletDB.prototype.removeBlock = function removeBlock(block, callback, force) {
+WalletDB.prototype.removeBlock = function removeBlock(entry, callback, force) {
   var self = this;
   var unlock;
 
-  unlock = this.locker.lock(removeBlock, [block, callback], force);
+  unlock = this.locker.lock(removeBlock, [entry, callback], force);
 
   if (!unlock)
     return;
 
   callback = utils.wrap(callback, unlock);
 
-  this.getWallets(function(err, wallets) {
+  // Note:
+  // If we crash during a reorg, there's not much to do.
+  // Reorgs cannot be rescanned. The database will be
+  // in an odd state, with some txs being confirmed
+  // when they shouldn't be. That being said, this
+  // should eventually resolve itself when a new block
+  // comes in.
+  this.getBlock(entry.hash, function(err, block) {
     if (err)
       return callback(err);
 
-    utils.forEachSerial(wallets, function(id, next) {
+    // Not a saved block, but we
-      self.get(id, function(err, wallet) {
+    // still want to reset the tip.
-        if (err)
+    if (!block)
-          return next(err);
+      block = WalletBlock.fromEntry(entry);
 
-        if (!wallet)
+    // Unwrite the tip as fast as we can.
-          return next();
+    self.unwriteBlock(block, function(err) {
-
-        wallet.tx.getHeightHashes(block.height, function(err, hashes) {
-          if (err)
-            return callback(err);
-
-          utils.forEachSerial(hashes, function(hash, next) {
-            wallet.tx.unconfirm(hash, next);
-          }, next);
-        });
-      });
-    }, function(err) {
       if (err)
         return callback(err);
-      self.setTip(block.prevBlock, block.height - 1, callback);
+
+      utils.forEachSerial(block.hashes, function(hash, next) {
+        self.getWalletsByTX(hash, function(err, wallets) {
+          if (err)
+            return next(err);
+
+          if (!wallets)
+            return next();
+
+          utils.forEachSerial(wallets, function(id, next) {
+            self.get(id, function(err, wallet) {
+              if (err)
+                return next(err);
+
+              if (!wallet)
+                return next();
+
+              wallet.tx.unconfirm(hash, next);
+            });
+          }, callback);
+        });
+      });
     });
   });
 };
@@ -1214,6 +1316,11 @@ WalletDB.prototype.removeBlock = function removeBlock(block, callback, force) {
 
 WalletDB.prototype.addTX = function addTX(tx, callback, force) {
   var self = this;
+
+  // Note:
+  // Atomicity doesn't matter here. If we crash,
+  // the automatic rescan will get the database
+  // back in the correct state.
   this.mapWallets(tx, function(err, wallets) {
     if (err)
       return callback(err);
@@ -1242,7 +1349,11 @@ WalletDB.prototype.addTX = function addTX(tx, callback, force) {
           wallet.handleTX(info, next);
         });
       });
-    }, callback);
+    }, function(err) {
+      if (err)
+        return callback(err);
+      return callback(null, wallets);
+    });
   });
 };
 
@@ -1630,6 +1741,108 @@ function serializePaths(out) {
   return p.render();
 }
 
+function serializeWallets(wallets) {
+  var p = new BufferWriter();
+  var i, info;
+
+  for (i = 0; i < wallets.length; i++) {
+    info = wallets[i];
+    p.writeVarBytes(info.id, 'ascii');
+  }
+
+  return p.render();
+}
+
+function parseWallets(data) {
+  var p = new BufferReader(data);
+  var wallets = [];
+  while (p.left())
+    wallets.push(p.readVarString('ascii'));
+  return wallets;
+}
+
+function WalletBlock(hash, height) {
+  if (!(this instanceof WalletBlock))
+    return new WalletBlock(hash, height);
+
+  this.hash = hash || constants.NULL_HASH;
+  this.height = height != null ? height : -1;
+  this.prevBlock = constants.NULL_HASH;
+  this.hashes = [];
+}
+
+WalletBlock.prototype.fromEntry = function fromEntry(entry) {
+  this.hash = entry.hash;
+  this.height = entry.height;
+  this.prevBlock = entry.prevBlock;
+  return this;
+};
+
+WalletBlock.prototype.fromJSON = function fromJSON(json) {
+  this.hash = utils.revHex(json.hash);
+  this.height = entry.height;
+  if (entry.prevBlock)
+    this.prevBlock = utils.revHex(entry.prevBlock);
+  return this;
+};
+
+WalletBlock.prototype.fromRaw = function fromRaw(hash, data) {
+  var p = new BufferReader(data);
+  this.hash = hash;
+  this.height = p.readU32();
+  while (p.left())
+    this.hashes.push(p.readHash('hex'));
+  return this;
+};
+
+WalletBlock.prototype.fromTip = function fromTip(data) {
+  this.hash = p.readHash('hex');
+  this.height = p.readU32();
+  return this;
+};
+
+WalletBlock.fromEntry = function fromEntry(entry) {
+  return new WalletBlock().fromEntry(entry);
+};
+
+WalletBlock.fromJSON = function fromJSON(json) {
+  return new WalletBlock().fromJSON(json);
+};
+
+WalletBlock.fromRaw = function fromRaw(hash, data) {
+  return new WalletBlock().fromTip(hash, data);
+};
+
+WalletBlock.fromTip = function fromTip(data) {
+  return new WalletBlock().fromTip(data);
+};
+
+WalletBlock.prototype.toTip = function toTip(data) {
+  var p = new BufferWriter();
+  p.writeHash(this.hash);
+  p.writeU32(this.height);
+  return p.render();
+};
+
+WalletBlock.prototype.toRaw = function toRaw(data) {
+  var p = new BufferWriter();
+  var i;
+
+  p.writeU32(this.height);
+
+  for (i = 0; i < this.hashes.length; i++)
+    p.writeHash(this.hashes[i]);
+
+  return p.render();
+};
+
+WalletBlock.prototype.toJSON = function toJSON() {
+  return {
+    hash: utils.revHex(this.hash),
+    height: this.height
+  };
+};
+
 function ReadLock(parent) {
   if (!(this instanceof ReadLock))
     return new ReadLock(parent);