walletdb: use bloom filter for rescanning.

lib/chain/chaindb.js

@@ -1099,15 +1099,16 @@ ChainDB.prototype.getTXByAddress = co(function* getTXByAddress(addresses) {
 /**
  * Scan the blockchain for transactions containing specified address hashes.
  * @param {Hash} start - Block hash to start at.
- * @param {Hash[]} hashes - Address hashes.
+ * @param {Bloom} filter - Bloom filter containing tx and address hashes.
  * @param {Function} iter - Iterator.
  * @returns {Promise}
  */
 
 ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
   var total = 0;
-  var i, j, entry, hashes, hash, tx, txs, block;
+  var i, j, entry, hashes;
-  var input, prevout, found, txid;
+  var hash, tx, txs, block;
+  var input, prevout, found;
 
   if (start == null)
     start = this.network.genesis.hash;
@@ -1117,9 +1118,6 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
   else
     this.logger.info('Scanning from block %s.', utils.revHex(start));
 
-  if (Array.isArray(filter))
-    filter = utils.toMap(filter);
-
   entry = yield this.getEntry(start);
 
   if (!entry)
@@ -1149,29 +1147,29 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
       tx = block.txs[i];
       found = false;
 
-      for (j = 0; j < tx.inputs.length; j++) {
+      if (!tx.isCoinbase()) {
-        input = tx.inputs[j];
+        for (j = 0; j < tx.inputs.length; j++) {
-        prevout = input.prevout;
+          input = tx.inputs[j];
-        if (filter[prevout.hash]) {
+          prevout = input.prevout;
-          txs.push(tx);
+          if (filter.test(prevout.hash, 'hex')) {
-          txid = tx.hash('hex');
+            txs.push(tx);
-          filter[txid] = true;
+            filter.add(tx.hash());
-          found = true;
+            found = true;
-          break;
+            break;
+          }
         }
+
+        if (found)
+          continue;
       }
 
-      if (found)
+      hashes = tx.getOutputHashes();
-        continue;
-
-      hashes = tx.getOutputHashes('hex');
 
       for (j = 0; j < hashes.length; j++) {
         hash = hashes[j];
-        if (filter[hash]) {
+        if (filter.test(hash)) {
           txs.push(tx);
-          txid = tx.hash('hex');
+          filter.add(tx.hash());
-          filter[txid] = true;
           break;
         }
       }

lib/http/server.js

@@ -17,6 +17,7 @@ var HTTPBase = require('./base');
 var utils = require('../utils/utils');
 var co = require('../utils/co');
 var Address = require('../primitives/address');
+var Bloom = require('../utils/bloom');
 var TX = require('../primitives/tx');
 var KeyRing = require('../primitives/keyring');
 var Outpoint = require('../primitives/outpoint');
@@ -1226,24 +1227,6 @@ HTTPServer.prototype._initIO = function _initIO() {
       callback();
     });
 
-    socket.on('unwatch hash', function(args, callback) {
-      var hashes = args[0];
-
-      if (!Array.isArray(hashes))
-        return callback({ error: 'Invalid parameter.' });
-
-      if (!socket.api)
-        return callback({ error: 'Not authorized.' });
-
-      try {
-        socket.removeFilter(hashes);
-      } catch (e) {
-        return callback({ error: e.message });
-      }
-
-      callback();
-    });
-
     socket.on('estimate fee', function(args, callback) {
       var blocks = args[0];
       var rate;
@@ -1433,8 +1416,7 @@ function ClientSocket(server, socket) {
   this.host = socket.conn.remoteAddress;
   this.timeout = null;
   this.auth = false;
-  this.filter = {};
+  this.filter = null;
-  this.filterCount = 0;
   this.api = false;
 
   this.node = this.server.node;
@@ -1483,23 +1465,8 @@ ClientSocket.prototype._init = function _init() {
 ClientSocket.prototype.addFilter = function addFilter(hashes) {
   var i, hash;
 
-  for (i = 0; i < hashes.length; i++) {
+  if (!this.filter)
-    hash = Address.getHash(hashes[i], 'hex');
+    this.filter = Bloom.fromRate(100000, 0.001, -1);
-
-    if (!hash)
-      throw new Error('Bad hash.');
-
-    if (!this.filter[hash]) {
-      this.filter[hash] = true;
-      this.filterCount++;
-      if (this.pool.options.spv)
-        this.pool.watch(hash, 'hex');
-    }
-  }
-};
-
-ClientSocket.prototype.removeFilter = function removeFilter(hashes) {
-  var i, hash;
 
   for (i = 0; i < hashes.length; i++) {
     hash = Address.getHash(hashes[i], 'hex');
@@ -1507,10 +1474,10 @@ ClientSocket.prototype.removeFilter = function removeFilter(hashes) {
     if (!hash)
       throw new Error('Bad hash.');
 
-    if (this.filter[hash]) {
+    this.filter.add(hash, 'hex');
-      delete this.filter[hash];
+
-      this.filterCount--;
+    if (this.pool.options.spv)
-    }
+      this.pool.watch(hash, 'hex');
   }
 };
 
@@ -1578,7 +1545,7 @@ ClientSocket.prototype.testBlock = function testBlock(block) {
   var txs = [];
   var i, tx;
 
-  if (this.filterCount === 0)
+  if (!this.filter)
     return;
 
   for (i = 0; i < block.txs.length; i++) {
@@ -1594,18 +1561,33 @@ ClientSocket.prototype.testBlock = function testBlock(block) {
 };
 
 ClientSocket.prototype.testFilter = function testFilter(tx) {
-  var i, hashes, hash;
+  var i, hashes, hash, input, prevout;
 
-  if (this.filterCount === 0)
+  if (!this.filter)
-    return;
+    return false;
 
-  hashes = tx.getHashes('hex');
+  if (!tx.isCoinbase()) {
+    for (i = 0; i < tx.inputs.length; i++) {
+      input = tx.inputs[i];
+      prevout = input.prevout;
+      if (this.filter.test(prevout.hash, 'hex')) {
+        this.filter.add(tx.hash());
+        return true;
+      }
+    }
+  }
+
+  hashes = tx.getOutputHashes();
 
   for (i = 0; i < hashes.length; i++) {
     hash = hashes[i];
-    if (this.filter[hash])
+    if (this.filter.test(hash)) {
+      this.filter.add(tx.hash());
       return true;
+    }
   }
+
+  return false;
 };
 
 ClientSocket.prototype.scan = co(function* scan(start) {

lib/wallet/walletdb.js

@@ -185,9 +185,7 @@ function WalletDB(options) {
   // lose membership, even if quality
   // degrades.
   // Memory used: 1.7mb
-  this.filter = this.options.useFilter !== false
+  this.filter = Bloom.fromRate(1000000, 0.001, -1);
-    ? Bloom.fromRate(1000000, 0.001, -1)
-    : null;
 
   this.db = ldb({
     location: this.options.location,
@@ -239,7 +237,6 @@ WalletDB.prototype._open = co(function* open() {
     'WalletDB loaded (depth=%d, height=%d).',
     this.depth, this.height);
 
-  yield this.loadFilter();
   yield this.connect();
   yield this.resend();
 });
@@ -285,18 +282,16 @@ WalletDB.prototype.connect = co(function* connect() {
  */
 
 WalletDB.prototype._connect = co(function* connect() {
-  var hashes, tip, height;
+  var hashes = yield this.getFilterHashes();
+  var tip, height;
+
+  this.logger.info('Adding %d hashes to filter.', hashes.length);
+
+  this.addFilter(hashes);
 
   if (!this.client)
     return;
 
-  // yield this.client.connect();
-  // yield this.client.watchChain();
-
-  hashes = yield this.getFilterHashes();
-
-  yield this.client.watchHash(hashes);
-
   if (this.options.noScan) {
     tip = yield this.client.getTip();
 
@@ -342,18 +337,10 @@ WalletDB.prototype.rescan = co(function* rescan(height) {
  */
 
 WalletDB.prototype._rescan = co(function* rescan(height) {
-  var hashes;
-
   if (!this.client)
     return;
 
-  assert(utils.isNumber(height), 'Must pass in a height.');
+  yield this.scan(height);
-
-  hashes = yield this.getFilterHashes();
-
-  yield this.client.watchHash(hashes);
-
-  yield this.scan(height, hashes);
 });
 
 /**
@@ -364,7 +351,7 @@ WalletDB.prototype._rescan = co(function* rescan(height) {
  * @returns {Promise}
  */
 
-WalletDB.prototype.scan = co(function* scan(height, hashes) {
+WalletDB.prototype.scan = co(function* scan(height) {
   var self = this;
   var blocks;
 
@@ -383,9 +370,9 @@ WalletDB.prototype.scan = co(function* scan(height, hashes) {
 
   yield this.rollback(height);
 
-  this.logger.info('Scanning for %d hashes.', hashes.length);
+  this.logger.info('Scanning for blocks.');
 
-  yield this.client.scan(this.tip.hash, hashes, function(block, txs) {
+  yield this.client.scan(this.tip.hash, this.filter, function(block, txs) {
     return self._addBlock(block, txs);
   });
 });
@@ -467,12 +454,12 @@ WalletDB.prototype.wipe = co(function* wipe() {
   }
 
   gte = new Buffer(33);
-  key.fill(0);
+  gte.fill(0);
-  key[0] = 0x62;
+  gte[0] = 0x62;
 
   lte = new Buffer(33);
-  key.fill(255);
+  lte.fill(255);
-  key[0] = 0x62;
+  lte[0] = 0x62;
 
   keys = yield this.db.keys({
     gte: gte,
@@ -648,26 +635,6 @@ WalletDB.prototype.commit = co(function* commit(wallet) {
   wallet.pathCache.commit();
 });
 
-/**
- * Load the bloom filter into memory.
- * @private
- * @returns {Promise}
- */
-
-WalletDB.prototype.loadFilter = co(function* loadFilter() {
-  var i, hashes, hash;
-
-  if (!this.filter)
-    return;
-
-  hashes = yield this.getFilterHashes();
-
-  for (i = 0; i < hashes.length; i++) {
-    hash = hashes[i];
-    this.filter.add(hash, 'hex');
-  }
-});
-
 /**
  * Test the bloom filter against a tx or address hash.
  * @private
@@ -676,9 +643,6 @@ WalletDB.prototype.loadFilter = co(function* loadFilter() {
  */
 
 WalletDB.prototype.testFilter = function testFilter(hash) {
-  if (!this.filter)
-    return true;
-
   return this.filter.test(hash, 'hex');
 };
 
@@ -688,11 +652,19 @@ WalletDB.prototype.testFilter = function testFilter(hash) {
  * @param {Hash} hash
  */
 
-WalletDB.prototype.addFilter = function addFilter(hash) {
+WalletDB.prototype.addFilter = function addFilter(hashes) {
-  if (!this.filter)
+  var i, hash;
-    return;
 
-  return this.filter.add(hash, 'hex');
+  if (!Array.isArray(hashes))
+    hashes = [hashes];
+
+  if (this.client)
+    this.client.watchHash(hashes);
+
+  for (i = 0; i < hashes.length; i++) {
+    hash = hashes[i];
+    this.filter.add(hash, 'hex');
+  }
 };
 
 /**
@@ -1138,8 +1110,6 @@ WalletDB.prototype.savePath = co(function* savePath(wallet, path) {
 
   this.addFilter(hash);
 
-  yield this.watchHash([hash]);
-
   map = yield this.getPathMap(hash);
 
   if (!map)
@@ -1660,7 +1630,6 @@ WalletDB.prototype.writeTXMap = function writeTXMap(wallet, hash, map) {
   var batch = this.batch(wallet);
   batch.put(layout.e(hash), map.toRaw());
   this.addFilter(hash);
-  this.watchHash([hash]);
 };
 
 /**

test/chain-test.js

@@ -235,9 +235,8 @@ describe('Chain', function() {
 
   it('should rescan for transactions', cob(function* () {
     var total = 0;
-    var hashes = yield walletdb.getHashes();
 
-    yield chain.db.scan(null, hashes, function(block, txs) {
+    yield chain.db.scan(null, walletdb.filter, function(block, txs) {
       total += txs.length;
       return Promise.resolve();
     });