diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js index 41753c2f..4dde2840 100644 --- a/lib/bcoin/blockdb.js +++ b/lib/bcoin/blockdb.js @@ -37,9 +37,12 @@ function BlockDB(options) { this.parser = new bcoin.protocol.parser(); - this.data = new BlockData(); - this.unspentCache = new bcoin.lru(32 * 1024 * 1024); - this.txCache = new bcoin.lru(32 * 1024 * 1024); + this.data = new BlockData(options); + + this.cache = { + unspent: new bcoin.lru(32 * 1024 * 1024), + tx: new bcoin.lru(32 * 1024 * 1024) + }; this.index = new levelup(this.file, { keyEncoding: 'ascii', @@ -49,9 +52,12 @@ function BlockDB(options) { compression: true, cacheSize: 16 * 1024 * 1024, writeBufferSize: 8 * 1024 * 1024, + // https://leveldb.googlecode.com/git-history/master/doc/index.html + // Higher block size = better for iterators + // Lower block size = better for gets // blockSize: 4 * 1024, - // blockRestartInterval: 16, maxOpenFiles: 8192, + // blockRestartInterval: 16, db: bcoin.isBrowser ? require('memdown') : require('level' + 'down') @@ -60,6 +66,56 @@ function BlockDB(options) { inherits(BlockDB, EventEmitter); +BlockDB.prototype.migrate = function migrate(blockSize, compression, callback) { + var options, db, pending, stream, total, done; + + options = utils.merge({}, this.index.options); + + if (blockSize != null) + options.blockSize = blockSize; + + if (compression != null) + options.compression = compression; + + utils.print('Migrating DB with options:'); + utils.print(options); + + db = levelup(this.file + '.migrated', options); + + stream = this.index.createReadStream(); + + pending = 0; + total = 0; + + function onPut(err) { + if (err) + return callback(err); + + if (++total % 10000 === 0) + utils.print('%d written.', total); + + pending--; + + if (done && !pending) + callback(); + } + + stream.on('data', function(data) { + pending++; + db.put(data.key, data.value, onPut); + }); + + stream.on('error', function(err) { + callback(err); + }); + + stream.on('end', function() { + done = true; + if (!pending) + callback(); + }); +}; + BlockDB.prototype.createOffset = function createOffset(size, offset, height) { var buf = new Buffer(16); utils.writeU32(buf, size, 0); @@ -138,6 +194,9 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) { } batch.del('u/t/' + input.prevout.hash + '/' + input.prevout.index); + + if (self.options.cache) + self.cache.unspent.remove(input.prevout.hash + '/' + input.prevout.index); }); tx.outputs.forEach(function(output) { @@ -213,7 +272,7 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { batch.del('t/t/' + hash); - self.fillTX2(tx, function(err) { + self.fillTX(tx, function(err) { if (err) return callback(err); @@ -289,7 +348,47 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { }); }; -BlockDB.prototype.fillTX = function fillTX(tx, callback) { +BlockDB.prototype.fillCoins = function fillCoins(txs, callback) { + var self = this; + var pending = txs.length; + + callback = utils.asyncify(callback); + + if (!pending) + return callback(); + + txs.forEach(function(tx) { + self.fillCoin(tx, function(err) { + if (err) + return callback(err); + + if (!--pending) + callback(); + }); + }); +}; + +BlockDB.prototype.fillTXs = function fillTXs(txs, callback) { + var self = this; + var pending = txs.length; + + callback = utils.asyncify(callback); + + if (!pending) + return callback(); + + txs.forEach(function(tx) { + self.fillTX(tx, function(err) { + if (err) + return callback(err); + + if (!--pending) + callback(); + }); + }); +}; + +BlockDB.prototype.fillCoin = function fillCoin(tx, callback) { var self = this; var pending = tx.inputs.length; @@ -317,7 +416,7 @@ BlockDB.prototype.fillTX = function fillTX(tx, callback) { }); }; -BlockDB.prototype.fillTX2 = function fillTX2(tx, callback) { +BlockDB.prototype.fillTX = function fillTX(tx, callback) { var self = this; var pending = tx.inputs.length; @@ -416,9 +515,11 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call var index = +parts[1]; var id = hash + '/' + index; var record = self.parseOffset(data.value); + pending++; - if (self.unspentCache.has(id)) { - coins.push(self.unspentCache.get(id)); + + if (self.options.cache && self.cache.unspent.has(id)) { + coins.push(self.cache.unspent.get(id)); pending--; if (done) { if (!pending) @@ -426,6 +527,7 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call } return; } + self.data.getAsync(record.size, record.offset, function(err, data) { var coin; @@ -438,6 +540,7 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call } catch (e) { return callback(e); } + coin = bcoin.coin({ version: 1, hash: hash, @@ -447,7 +550,10 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call value: data.value, spent: false }); - self.unspentCache.set(id, coin); + + if (self.options.cache) + self.cache.unspent.set(id, coin); + coins.push(coin); } @@ -564,8 +670,8 @@ BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback) pending++; - if (self.txCache.has(hash)) { - coins.push(self.txCache.get(hash)); + if (self.options.cache && self.cache.tx.has(hash)) { + coins.push(self.cache.tx.get(hash)); pending--; if (done) { if (!pending) @@ -588,11 +694,17 @@ BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback) } catch (e) { return callback(e); } + tx.height = record.height; tx.ts = entry.ts; tx.block = entry.hash; - self.txCache.set(hash, tx); txs.push(tx); + + if (self.options.cache) + self.cache.tx.set(hash, tx); + + if (self.options.paranoid && tx.hash('hex') !== hash) + return callback(new Error('BlockDB is corrupt. All is lost.')); } pending--; @@ -645,6 +757,8 @@ BlockDB.prototype.getTX = function getTX(hash, callback) { tx.height = record.height; tx.ts = entry.ts; tx.block = entry.hash; + if (self.options.paranoid && tx.hash('hex') !== hash) + return callback(new Error('BlockDB is corrupt. All is lost.')); } return callback(null, tx); @@ -682,6 +796,16 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) { } block._fileOffset = record.offset; block.height = record.height; + if (self.options.paranoid) { + if (typeof hash === 'number') { + hash = bcoin.chain.global.get(hash); + if (!hash) + return callback(null, block); + hash = hash.hash; + } + if (block.hash('hex') !== hash) + return callback(new Error('BlockDB is corrupt. All is lost.')); + } } return callback(null, block); @@ -689,6 +813,15 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) { }); }; +BlockDB.prototype.isSpent = function isSpent(hash, index, callback) { + this.getCoin(hash, index, function(err, coin) { + if (err) + return callback(err); + + return callback(null, coin ? false : true); + }); +}; + /** * BlockData */ diff --git a/lib/bcoin/lru.js b/lib/bcoin/lru.js index b50c300e..5117aece 100644 --- a/lib/bcoin/lru.js +++ b/lib/bcoin/lru.js @@ -12,13 +12,15 @@ var assert = utils.assert; * LRU */ -function LRU(maxSize) { +function LRU(maxSize, getSize) { if (!(this instanceof LRU)) - return new LRU(maxSize); + return new LRU(maxSize, getSize); this.data = {}; this.size = 0; this.maxSize = maxSize; + this.getSize = getSize; + this.head = null; this.tail = null; } @@ -26,6 +28,9 @@ function LRU(maxSize) { LRU.prototype._getSize = function _getSize(item) { var keySize = item.key.length * 2; + if (this.getSize) + return keySize + this.getSize(item.value); + if (item.value == null) return keySize + 1; @@ -129,6 +134,7 @@ LRU.prototype._appendList = function appendList(item) { LRU.prototype._insertList = function insertList(ref, item) { assert(!item.next); assert(!item.prev); + if (ref == null) { if (!this.head) { this.head = item; @@ -140,9 +146,11 @@ LRU.prototype._insertList = function insertList(ref, item) { } return; } + item.next = ref.next; item.prev = ref; ref.next = item; + if (ref === this.tail) this.tail = item; }; @@ -160,6 +168,12 @@ LRU.prototype._removeList = function removeList(item) { if (item == this.tail) this.tail = item.prev || this.head; + if (!this.head) + assert(!this.tail); + + if (!this.tail) + assert(!this.head); + item.prev = null; item.next = null; }; @@ -167,6 +181,7 @@ LRU.prototype._removeList = function removeList(item) { LRU.prototype._keys = function _keys() { var keys = []; var item; + for (item = this.head; item; item = item.next) { if (item === this.head) assert(!item.prev); @@ -176,6 +191,7 @@ LRU.prototype._keys = function _keys() { assert(item === this.tail); keys.push(item.key); } + return keys; }; diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index aa1a3109..122cfc0d 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -141,6 +141,29 @@ Mempool.prototype.getTXByAddress = function getTXByAddress(addresses) { return txs; }; +Mempool.prototype.fillCoin = +Mempool.prototype.fillTX = function fillTX(tx) { + var i, input, total; + + total = 0; + + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + + if (input.output) { + total++; + continue; + } + + if (this.hasTX(input.prevout.hash)) { + input.output = this.getCoin(input.prevout.hash, input.prevout.index); + total++; + } + } + + return total === tx.inputs.length; +}; + Mempool.prototype.getAll = function getAll(hash) { return Object.keys(this.txs).map(function(key) { return this.txs[key]; @@ -175,7 +198,7 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback) { this._lockTX(tx); - this.storage.fillTX(tx, function(err) { + this.storage.fillCoin(tx, function(err) { var i, input, dup, height, ts, priority; self._unlockTX(tx); diff --git a/lib/bcoin/node.js b/lib/bcoin/node.js index 1cd8d1d2..a4ea2542 100644 --- a/lib/bcoin/node.js +++ b/lib/bcoin/node.js @@ -35,7 +35,7 @@ function Node(options) { if (this.options.network) network.set(this.options.network); - this.storage = null; + this.block = null; this.mempool = null; this.pool = null; this.chain = null; @@ -55,13 +55,13 @@ Node.prototype._init = function _init() { this.options.pool.type = 'full'; - this.storage = new bcoin.blockdb(this.options.storage); + this.block = new bcoin.blockdb(this.options.block); this.mempool = new bcoin.mempool(this, this.options.mempool); this.pool = new bcoin.pool(this.options.pool); this.chain = this.pool.chain; this.pool.on('block', function(block, peer) { - self.storage.saveBlock(block, function(err) { + self.block.saveBlock(block, function(err) { if (err) throw err; @@ -69,12 +69,12 @@ Node.prototype._init = function _init() { if (0) var hash = block.txs[0].hash('hex'); if (0) - self.storage.getTX(hash, function(err, tx) { + self.block.getTX(hash, function(err, tx) { if (err) throw err; utils.print(tx); }); if (0) - self.storage.getCoin(hash, 0, function(err, tx) { + self.block.getCoin(hash, 0, function(err, tx) { if (err) throw err; utils.print(tx); }); @@ -95,7 +95,7 @@ Node.prototype._init = function _init() { this.pool.on('fork', function(a, b) { [a, b].forEach(function(hash) { - self.storage.removeBlock(hash, function(err, block) { + self.block.removeBlock(hash, function(err, block) { if (err) throw err; @@ -128,7 +128,7 @@ Node.prototype.getCoin = function getCoin(hash, index, callback) { if (this.mempool.isSpent(hash, index)) return callback(null, null); - this.storage.getCoin(hash, index, function(err, coin) { + this.block.getCoin(hash, index, function(err, coin) { if (err) return callback(err); @@ -147,7 +147,7 @@ Node.prototype.getCoinByAddress = function getCoinsByAddress(addresses, callback mempool = this.mempool.getCoinsByAddress(addresses); - this.storage.getCoinsByAddress(addresses, function(err, coins) { + this.block.getCoinsByAddress(addresses, function(err, coins) { if (err) return callback(err); @@ -169,7 +169,7 @@ Node.prototype.getTX = function getTX(hash, callback) { if (tx) return callback(null, tx); - this.storage.getTX(hash, function(err, tx) { + this.block.getTX(hash, function(err, tx) { if (err) return callback(err); @@ -185,12 +185,7 @@ Node.prototype.isSpent = function isSpent(hash, index, callback) { if (this.mempool.isSpent(hash, index)) return callback(null, true); - this.storage.getCoin(hash, index, function(err, coin) { - if (err) - return callback(err); - - return callback(null, coin ? false : true); - }); + this.block.isSpent(hash, index, callback); }; Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) { @@ -201,7 +196,7 @@ Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) { mempool = this.mempool.getTXByAddress(addresses); - this.storage.getTXByAddress(addresses, function(err, txs) { + this.block.getTXByAddress(addresses, function(err, txs) { if (err) return callback(err); @@ -209,8 +204,18 @@ Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) { }); }; +Node.prototype.fillCoin = function fillCoin(tx, callback) { + callback = utils.asyncify(callback); + if (this.mempool.fillCoin(tx)) + return callback(); + this.block.fillCoin(tx, callback); +}; + Node.prototype.fillTX = function fillTX(tx, callback) { - this.storage.fillTX(tx, callback); + callback = utils.asyncify(callback); + if (this.mempool.fillTX(tx)) + return callback(); + this.block.fillTX(tx, callback); }; /** diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index 79857788..c330648a 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -582,6 +582,14 @@ Pool.prototype._handleInv = function _handleInv(hashes, peer) { } }; +Pool.prototype._prehandleTX = function _prehandleTX(tx, peer, callback) { + return callback(null, tx); +}; + +Pool.prototype._prehandleBlock = function _prehandleBlock(block, peer, callback) { + return callback(null, block); +}; + Pool.prototype._handleBlock = function _handleBlock(block, peer) { var self = this; var requested;