wallet/net: ensure low FPR of bloom filters. see #118.

lib/env.js

@@ -126,6 +126,7 @@ function Environment() {
   // Utils
   this.require('utils', './utils');
   this.require('base58', './utils/base58');
+  this.require('bloom', './utils/bloom');
   this.require('co', './utils/co');
   this.require('encoding', './utils/encoding');
   this.require('lock', './utils/lock');

lib/net/pool.js

@@ -122,7 +122,7 @@ function Pool(options) {
   this.hosts.address = this.address;
 
   if (this.options.spv)
-    this.spvFilter = Bloom.fromRate(10000, 0.001, Bloom.flags.ALL);
+    this.spvFilter = Bloom.fromRate(20000, 0.001, Bloom.flags.ALL);
 
   if (!this.options.mempool)
     this.txFilter = new Bloom.Rolling(50000, 0.000001);

lib/primitives/merkleblock.js

@@ -163,9 +163,14 @@ MerkleBlock.prototype.verifyPartial = function verifyPartial() {
   if (this._validPartial != null)
     return this._validPartial;
 
-  tree = this.extractTree();
+  try {
+    tree = this.extractTree();
+  } catch (e) {
+    this._validPartial = false;
+    return false;
+  }
 
-  if (!tree || tree.root !== this.merkleRoot) {
+  if (tree.root !== this.merkleRoot) {
     this._validPartial = false;
     return false;
   }
@@ -246,16 +251,16 @@ MerkleBlock.prototype.extractTree = function extractTree() {
     hashes.push(this.hashes[p]);
 
   if (totalTX === 0)
-    return;
+    throw new Error('Zero transactions.');
 
   if (totalTX > consensus.MAX_BLOCK_SIZE / 60)
-    return;
+    throw new Error('Too many transactions.');
 
   if (hashes.length > totalTX)
-    return;
+    throw new Error('Too many hashes.');
 
   if (flags.length * 8 < hashes.length)
-    return;
+    throw new Error('Flags too small.');
 
   while (width(height) > 1)
     height++;
@@ -266,13 +271,13 @@ MerkleBlock.prototype.extractTree = function extractTree() {
   root = traverse(height, 0);
 
   if (failed)
-    return;
+    throw new Error('Mutated merkle tree.');
 
   if (((bitsUsed + 7) / 8 | 0) !== flags.length)
-    return;
+    throw new Error('Too many flag bits.');
 
   if (hashUsed !== hashes.length)
-    return;
+    throw new Error('Incorrect number of hashes.');
 
   return new PartialTree(root, matches, indexes, map);
 };

lib/utils/bloom.js

@@ -274,13 +274,17 @@ Bloom.fromRate = function fromRate(items, rate, update) {
   size = (-1 / LN2SQUARED * items * Math.log(rate)) | 0;
   size = Math.max(8, size);
 
-  if (update !== -1)
-    size = Math.min(size, Bloom.MAX_BLOOM_FILTER_SIZE * 8);
+  if (update !== -1) {
+    assert(size <= Bloom.MAX_BLOOM_FILTER_SIZE * 8,
+      'Bloom filter size violates policy limits!');
+  }
 
   n = Math.max(1, (size / items * LN2) | 0);
 
-  if (update !== -1)
-    n = Math.min(n, Bloom.MAX_HASH_FUNCS);
+  if (update !== -1) {
+    assert(n <= Bloom.MAX_HASH_FUNCS,
+      'Bloom filter size violates policy limits!');
+  }
 
   return new Bloom(size, n, -1, update);
 };

lib/wallet/walletdb.js

@@ -76,7 +76,9 @@ function WalletDB(options) {
   this.widCache = new LRU(10000);
   this.pathMapCache = new LRU(100000);
 
-  this.filter = Bloom.fromRate(1000000, 0.001, this.options.spv ? 1 : -1);
+  this.filter = new Bloom();
+
+  this._init();
 }
 
 util.inherits(WalletDB, AsyncObject);
@@ -88,6 +90,28 @@ util.inherits(WalletDB, AsyncObject);
 
 WalletDB.layout = layout;
 
+/**
+ * Initialize walletdb.
+ * @private
+ */
+
+WalletDB.prototype._init = function _init() {
+  var items = 1000000;
+  var flag = -1;
+
+  // Highest number of items with an
+  // FPR of 0.001. We have to do this
+  // by hand because Bloom.fromRate's
+  // policy limit enforcing is fairly
+  // naive.
+  if (this.options.spv) {
+    items = 20000;
+    flag = Bloom.flags.ALL;
+  }
+
+  this.filter = Bloom.fromRate(items, 0.001, flag);
+};
+
 /**
  * Open the walletdb, wait for the database to load.
  * @alias WalletDB#open