RanchiMall/fcoin
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

walletdb: use bloom filter for rescanning.

Browse Source
This commit is contained in:
Christopher Jeffrey 2016-10-24 18:27:25 -07:00
parent 3bffbed25a
commit 5a4ebccfd4
No known key found for this signature in database
GPG Key ID: 8962AB9DE6666BBD
4 changed files with 77 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
lib/chain/chaindb.js
View File

@ -1099,15 +1099,16 @@ ChainDB.prototype.getTXByAddress = co(function* getTXByAddress(addresses) {
/**
* Scan the blockchain for transactions containing specified address hashes.
* @param {Hash} start - Block hash to start at.
* @param {Hash[]} hashes - Address hashes.
* @param {Bloom} filter - Bloom filter containing tx and address hashes.
* @param {Function} iter - Iterator.
* @returns {Promise}
*/
ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
var total = 0;
var i, j, entry, hashes, hash, tx, txs, block;
var input, prevout, found, txid;
var i, j, entry, hashes;
var hash, tx, txs, block;
var input, prevout, found;
if (start == null)
start = this.network.genesis.hash;
@ -1117,9 +1118,6 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
else
this.logger.info('Scanning from block %s.', utils.revHex(start));
if (Array.isArray(filter))
filter = utils.toMap(filter);
entry = yield this.getEntry(start);
if (!entry)
@ -1149,29 +1147,29 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
tx = block.txs[i];
found = false;
for (j = 0; j < tx.inputs.length; j++) {
input = tx.inputs[j];
prevout = input.prevout;
if (filter[prevout.hash]) {
txs.push(tx);
txid = tx.hash('hex');
filter[txid] = true;
found = true;
break;
if (!tx.isCoinbase()) {
for (j = 0; j < tx.inputs.length; j++) {
input = tx.inputs[j];
prevout = input.prevout;
if (filter.test(prevout.hash, 'hex')) {
txs.push(tx);
filter.add(tx.hash());
found = true;
break;
}
}
if (found)
continue;
}
if (found)
continue;
hashes = tx.getOutputHashes('hex');
hashes = tx.getOutputHashes();
for (j = 0; j < hashes.length; j++) {
hash = hashes[j];
if (filter[hash]) {
if (filter.test(hash)) {
txs.push(tx);
txid = tx.hash('hex');
filter[txid] = true;
filter.add(tx.hash());
break;
}
}

76
lib/http/server.js
View File

@ -17,6 +17,7 @@ var HTTPBase = require('./base');
var utils = require('../utils/utils');
var co = require('../utils/co');
var Address = require('../primitives/address');
var Bloom = require('../utils/bloom');
var TX = require('../primitives/tx');
var KeyRing = require('../primitives/keyring');
var Outpoint = require('../primitives/outpoint');
@ -1226,24 +1227,6 @@ HTTPServer.prototype._initIO = function _initIO() {
callback();
});
socket.on('unwatch hash', function(args, callback) {
var hashes = args[0];
if (!Array.isArray(hashes))
return callback({ error: 'Invalid parameter.' });
if (!socket.api)
return callback({ error: 'Not authorized.' });
try {
socket.removeFilter(hashes);
} catch (e) {
return callback({ error: e.message });
}
callback();
});
socket.on('estimate fee', function(args, callback) {
var blocks = args[0];
var rate;
@ -1433,8 +1416,7 @@ function ClientSocket(server, socket) {
this.host = socket.conn.remoteAddress;
this.timeout = null;
this.auth = false;
this.filter = {};
this.filterCount = 0;
this.filter = null;
this.api = false;
this.node = this.server.node;
@ -1483,23 +1465,8 @@ ClientSocket.prototype._init = function _init() {
ClientSocket.prototype.addFilter = function addFilter(hashes) {
var i, hash;
for (i = 0; i < hashes.length; i++) {
hash = Address.getHash(hashes[i], 'hex');
if (!hash)
throw new Error('Bad hash.');
if (!this.filter[hash]) {
this.filter[hash] = true;
this.filterCount++;
if (this.pool.options.spv)
this.pool.watch(hash, 'hex');
}
}
};
ClientSocket.prototype.removeFilter = function removeFilter(hashes) {
var i, hash;
if (!this.filter)
this.filter = Bloom.fromRate(100000, 0.001, -1);
for (i = 0; i < hashes.length; i++) {
hash = Address.getHash(hashes[i], 'hex');
@ -1507,10 +1474,10 @@ ClientSocket.prototype.removeFilter = function removeFilter(hashes) {
if (!hash)
throw new Error('Bad hash.');
if (this.filter[hash]) {
delete this.filter[hash];
this.filterCount--;
}
this.filter.add(hash, 'hex');
if (this.pool.options.spv)
this.pool.watch(hash, 'hex');
}
};
@ -1578,7 +1545,7 @@ ClientSocket.prototype.testBlock = function testBlock(block) {
var txs = [];
var i, tx;
if (this.filterCount === 0)
if (!this.filter)
return;
for (i = 0; i < block.txs.length; i++) {
@ -1594,18 +1561,33 @@ ClientSocket.prototype.testBlock = function testBlock(block) {
};
ClientSocket.prototype.testFilter = function testFilter(tx) {
var i, hashes, hash;
var i, hashes, hash, input, prevout;
if (this.filterCount === 0)
return;
if (!this.filter)
return false;
hashes = tx.getHashes('hex');
if (!tx.isCoinbase()) {
for (i = 0; i < tx.inputs.length; i++) {
input = tx.inputs[i];
prevout = input.prevout;
if (this.filter.test(prevout.hash, 'hex')) {
this.filter.add(tx.hash());
return true;
}
}
}
hashes = tx.getOutputHashes();
for (i = 0; i < hashes.length; i++) {
hash = hashes[i];
if (this.filter[hash])
if (this.filter.test(hash)) {
this.filter.add(tx.hash());
return true;
}
}
return false;
};
ClientSocket.prototype.scan = co(function* scan(start) {

85
lib/wallet/walletdb.js
View File

@ -185,9 +185,7 @@ function WalletDB(options) {
// lose membership, even if quality
// degrades.
// Memory used: 1.7mb
this.filter = this.options.useFilter !== false
? Bloom.fromRate(1000000, 0.001, -1)
: null;
this.filter = Bloom.fromRate(1000000, 0.001, -1);
this.db = ldb({
location: this.options.location,
@ -239,7 +237,6 @@ WalletDB.prototype._open = co(function* open() {
'WalletDB loaded (depth=%d, height=%d).',
this.depth, this.height);
yield this.loadFilter();
yield this.connect();
yield this.resend();
});
@ -285,18 +282,16 @@ WalletDB.prototype.connect = co(function* connect() {
*/
WalletDB.prototype._connect = co(function* connect() {
var hashes, tip, height;
var hashes = yield this.getFilterHashes();
var tip, height;
this.logger.info('Adding %d hashes to filter.', hashes.length);
this.addFilter(hashes);
if (!this.client)
return;
// yield this.client.connect();
// yield this.client.watchChain();
hashes = yield this.getFilterHashes();
yield this.client.watchHash(hashes);
if (this.options.noScan) {
tip = yield this.client.getTip();
@ -342,18 +337,10 @@ WalletDB.prototype.rescan = co(function* rescan(height) {
*/
WalletDB.prototype._rescan = co(function* rescan(height) {
var hashes;
if (!this.client)
return;
assert(utils.isNumber(height), 'Must pass in a height.');
hashes = yield this.getFilterHashes();
yield this.client.watchHash(hashes);
yield this.scan(height, hashes);
yield this.scan(height);
});
/**
@ -364,7 +351,7 @@ WalletDB.prototype._rescan = co(function* rescan(height) {
* @returns {Promise}
*/
WalletDB.prototype.scan = co(function* scan(height, hashes) {
WalletDB.prototype.scan = co(function* scan(height) {
var self = this;
var blocks;
@ -383,9 +370,9 @@ WalletDB.prototype.scan = co(function* scan(height, hashes) {
yield this.rollback(height);
this.logger.info('Scanning for %d hashes.', hashes.length);
this.logger.info('Scanning for blocks.');
yield this.client.scan(this.tip.hash, hashes, function(block, txs) {
yield this.client.scan(this.tip.hash, this.filter, function(block, txs) {
return self._addBlock(block, txs);
});
});
@ -467,12 +454,12 @@ WalletDB.prototype.wipe = co(function* wipe() {
}
gte = new Buffer(33);
key.fill(0);
key[0] = 0x62;
gte.fill(0);
gte[0] = 0x62;
lte = new Buffer(33);
key.fill(255);
key[0] = 0x62;
lte.fill(255);
lte[0] = 0x62;
keys = yield this.db.keys({
gte: gte,
@ -648,26 +635,6 @@ WalletDB.prototype.commit = co(function* commit(wallet) {
wallet.pathCache.commit();
});
/**
* Load the bloom filter into memory.
* @private
* @returns {Promise}
*/
WalletDB.prototype.loadFilter = co(function* loadFilter() {
var i, hashes, hash;
if (!this.filter)
return;
hashes = yield this.getFilterHashes();
for (i = 0; i < hashes.length; i++) {
hash = hashes[i];
this.filter.add(hash, 'hex');
}
});
/**
* Test the bloom filter against a tx or address hash.
* @private
@ -676,9 +643,6 @@ WalletDB.prototype.loadFilter = co(function* loadFilter() {
*/
WalletDB.prototype.testFilter = function testFilter(hash) {
if (!this.filter)
return true;
return this.filter.test(hash, 'hex');
};
@ -688,11 +652,19 @@ WalletDB.prototype.testFilter = function testFilter(hash) {
* @param {Hash} hash
*/
WalletDB.prototype.addFilter = function addFilter(hash) {
if (!this.filter)
return;
WalletDB.prototype.addFilter = function addFilter(hashes) {
var i, hash;
return this.filter.add(hash, 'hex');
if (!Array.isArray(hashes))
hashes = [hashes];
if (this.client)
this.client.watchHash(hashes);
for (i = 0; i < hashes.length; i++) {
hash = hashes[i];
this.filter.add(hash, 'hex');
}
};
/**
@ -1138,8 +1110,6 @@ WalletDB.prototype.savePath = co(function* savePath(wallet, path) {
this.addFilter(hash);
yield this.watchHash([hash]);
map = yield this.getPathMap(hash);
if (!map)
@ -1660,7 +1630,6 @@ WalletDB.prototype.writeTXMap = function writeTXMap(wallet, hash, map) {
var batch = this.batch(wallet);
batch.put(layout.e(hash), map.toRaw());
this.addFilter(hash);
this.watchHash([hash]);
};
/**

3
test/chain-test.js
View File

@ -235,9 +235,8 @@ describe('Chain', function() {
it('should rescan for transactions', cob(function* () {
var total = 0;
var hashes = yield walletdb.getHashes();
yield chain.db.scan(null, hashes, function(block, txs) {
yield chain.db.scan(null, walletdb.filter, function(block, txs) {
total += txs.length;
return Promise.resolve();
});