diff --git a/lib/bcoin.js b/lib/bcoin.js index 7c8b34dd..be8aa950 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -92,7 +92,6 @@ bcoin.block = require('./bcoin/block'); bcoin.merkleblock = require('./bcoin/merkleblock'); bcoin.headers = require('./bcoin/headers'); bcoin.ramdisk = require('./bcoin/ramdisk'); -bcoin.blockdb = require('./bcoin/blockdb'); bcoin.node = require('./bcoin/node'); bcoin.spvnode = require('./bcoin/spvnode'); bcoin.fullnode = require('./bcoin/fullnode'); diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js deleted file mode 100644 index d3db60b1..00000000 --- a/lib/bcoin/blockdb.js +++ /dev/null @@ -1,995 +0,0 @@ -/** - * db.js - db object for bcoin - * Copyright (c) 2014-2015, Fedor Indutny (MIT License) - * https://github.com/indutny/bcoin - */ - -var bcoin = require('../bcoin'); -var utils = bcoin.utils; -var assert = utils.assert; -var EventEmitter = require('events').EventEmitter; -var network = bcoin.protocol.network; -var DUMMY = new Buffer([]); -var pad32 = utils.pad32; - -/** - * BlockDB - */ - -function BlockDB(options, db) { - var self = this; - - if (!(this instanceof BlockDB)) - return new BlockDB(options); - - EventEmitter.call(this); - - if (!options) - options = {}; - - this.options = options; - this.fsync = !!options.fsync; - this.keepBlocks = options.keepBlocks || 288; - this.prune = !!options.prune; - - this.db = db; -} - -utils.inherits(BlockDB, EventEmitter); - -BlockDB.prototype.close = function close(callback) { - var self = this; - this.db.close(function(err) { - if (err) - return callback(err); - - return callback(); - }); -}; - -BlockDB.prototype.saveBlock = function saveBlock(block, batch, callback) { - var self = this; - - batch.put('b/b/' + block.hash('hex'), block.toCompact()); - - block.txs.forEach(function(tx, i) { - batch.put('t/t/' + tx.hash('hex'), tx.toExtended()); - }); - - self.connectBlock(block, batch, callback); - return; - self.connectBlock(block, batch, function(err) { - if (err) - return callback(err); - - if (!self.prune) - return callback(null, block); - - // Check for now-fully-spent txs. Try to remove - // them or queue them up for future deletion if - // it is currently unsafe to remove them. - self._pruneBlock(block, batch, function(err) { - if (err) - return callback(err); - - return callback(null, block); - }); - }); -}; - -BlockDB.prototype.removeBlock = function removeBlock(hash, batch, callback) { - var self = this; - - this._getTXBlock(hash, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(); - - batch.del('b/b/' + block.hash('hex')); - - block.txs.forEach(function(tx, i) { - batch.del('t/t/' + tx.hash('hex')); - }); - - self.disconnectBlock(block, batch, callback); - }); -}; - -BlockDB.prototype.connectBlock = function connectBlock(block, batch, callback) { - var self = this; - - this._getCoinBlock(block, function(err, block) { - var height; - - if (err) - return callback(err); - - if (!block) - return callback(); - - block.txs.forEach(function(tx, i) { - var hash = tx.hash('hex'); - var uniq = {}; - - tx.inputs.forEach(function(input) { - var address; - - if (input.isCoinbase()) - return; - - assert(input.output); - - if (self.options.indexAddress) { - address = input.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.put('t/a/' + address + '/' + hash, DUMMY); - } - - if (address) { - batch.del( - 'u/a/' + address - + '/' + input.prevout.hash - + '/' + input.prevout.index); - } - } - - batch.del('u/t/' + input.prevout.hash + '/' + input.prevout.index); - }); - - tx.outputs.forEach(function(output, i) { - var address; - - if (self.options.indexAddress) { - address = output.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.put('t/a/' + address + '/' + hash, DUMMY); - } - - if (address) - batch.put('u/a/' + address + '/' + hash + '/' + i, DUMMY); - } - - batch.put('u/t/' + hash + '/' + i, bcoin.coin(tx, i).toExtended()); - }); - }); - - self.emit('add block', block); - - return callback(null, block); - }); -}; - -BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callback) { - var self = this; - - this._getTXBlock(hash, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(); - - if (typeof hash === 'string') - assert(block.hash('hex') === hash); - - block.txs.forEach(function(tx, i) { - var hash = tx.hash('hex'); - var uniq = {}; - - tx.inputs.forEach(function(input) { - var coin, address; - - if (input.isCoinbase()) - return; - - assert(input.output); - - if (self.options.indexAddress) { - address = input.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.del('t/a/' + address + '/' + hash); - } - - if (address) { - batch.put('u/a/' + address - + '/' + input.prevout.hash - + '/' + input.prevout.index, - DUMMY); - } - } - - batch.put('u/t/' - + input.prevout.hash - + '/' + input.prevout.index, - input.output.toExtended()); - }); - - tx.outputs.forEach(function(output, i) { - var address; - - if (self.options.indexAddress) { - address = output.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.del('t/a/' + address + '/' + hash); - } - - if (address) - batch.del('u/a/' + address + '/' + hash + '/' + i); - } - - batch.del('u/t/' + hash + '/' + i); - }); - }); - - self.emit('remove block', block); - return callback(null, block); - }); -}; - -BlockDB.prototype.fillCoin = function fillCoin(tx, callback) { - var self = this; - - callback = utils.asyncify(callback); - - if (Array.isArray(tx)) { - return utils.forEach(tx, function(tx, next) { - self.fillCoin(tx, next); - }, function(err) { - if (err) - return callback(err); - return callback(null, tx); - }); - } - - if (tx.isCoinbase()) - return callback(null, tx); - - utils.forEach(tx.inputs, function(input, next) { - if (input.output) - return next(); - - self.getCoin(input.prevout.hash, input.prevout.index, function(err, coin) { - if (err) - return callback(err); - - if (coin) - input.output = coin; - - next(); - }); - }, function(err) { - if (err) - return callback(err); - return callback(null, tx); - }); -}; - -BlockDB.prototype.fillTX = function fillTX(tx, callback) { - var self = this; - - callback = utils.asyncify(callback); - - if (Array.isArray(tx)) { - return utils.forEach(tx, function(tx, next) { - self.fillTX(tx, next); - }, function(err) { - if (err) - return callback(err); - return callback(null, tx); - }); - } - - if (tx.isCoinbase()) - return callback(null, tx); - - utils.forEach(tx.inputs, function(input, next) { - if (input.output) - return next(); - - self.getTX(input.prevout.hash, function(err, tx) { - if (err) - return next(err); - - if (tx) - input.output = bcoin.coin(tx, input.prevout.index); - - next(); - }); - }, function(err) { - if (err) - return callback(err); - return callback(null, tx); - }); -}; - -BlockDB.prototype.getCoinsByAddress = function getCoinsByAddress(addresses, options, callback) { - var self = this; - var ids = []; - var coins = []; - - if (!callback) { - callback = options; - options = {}; - } - - if (typeof addresses === 'string') - addresses = [addresses]; - - addresses = utils.uniqs(addresses); - - utils.forEach(addresses, function(address, done) { - var iter = self.db.db.iterator({ - gte: 'u/a/' + address, - lte: 'u/a/' + address + '~', - keys: true, - values: true, - fillCache: true, - keyAsBuffer: false, - valueAsBuffer: true - }); - - (function next() { - iter.next(function(err, key, value) { - var parts, hash, index; - - if (err) { - return iter.end(function() { - done(err); - }); - } - - if (key === undefined) { - return iter.end(function(err) { - if (err) - return done(err); - done(); - }); - } - - parts = key.split('/'); - hash = parts[3]; - index = +parts[4]; - - ids.push([hash, index]); - - next(); - }); - })(); - }, function(err) { - if (err) - return callback(err); - - utils.forEach(ids, function(item, next) { - var hash = item[0]; - var index = item[1]; - self.getCoin(hash, index, function(err, coin) { - if (err) - return next(err); - - if (!coin) - return next(); - - coins.push(coin); - next(); - }); - }, function(err) { - if (err) - return callback(err); - return callback(null, coins); - }); - }); -}; - -BlockDB.prototype.getCoin = function getCoin(hash, index, callback) { - var self = this; - var id = 'u/t/' + hash + '/' + index; - var coin; - - this.db.get(id, function(err, data) { - if (err) { - if (err.type === 'NotFoundError') - return callback(); - return callback(err); - } - - try { - coin = bcoin.coin.fromExtended(data); - } catch (e) { - return callback(e); - } - - return callback(null, coin); - }); -}; - -BlockDB.prototype.getTXByAddress = function getTXByAddress(addresses, options, callback) { - var self = this; - var hashes = []; - var txs = []; - var have = {}; - - if (!callback) { - callback = options; - options = {}; - } - - if (typeof addresses === 'string') - addresses = [addresses]; - - addresses = utils.uniqs(addresses); - - utils.forEach(addresses, function(address, done) { - var iter = self.db.db.iterator({ - gte: 't/a/' + address, - lte: 't/a/' + address + '~', - keys: true, - values: true, - fillCache: true, - keyAsBuffer: false, - valueAsBuffer: true - }); - - (function next() { - iter.next(function(err, key, value) { - var hash; - - if (err) { - return iter.end(function() { - done(err); - }); - } - - if (key === undefined) { - return iter.end(function(err) { - if (err) - return done(err); - done(); - }); - } - - hash = key.split('/')[3]; - - if (addresses.length > 1) { - if (have[hash]) - return next(); - - have[hash] = true; - } - - hashes.push(hash); - }); - })(); - }, function(err) { - utils.forEach(hashes, function(hash, next) { - self.getTX(hash, function(err, tx) { - if (err) - return next(err); - - if (!tx) - return next(); - - txs.push(tx); - next(); - }); - }, function(err) { - if (err) - return callback(err); - return callback(null, txs); - }); - }); -}; - -BlockDB.prototype.getTX = function getTX(hash, callback) { - var self = this; - var id = 't/t/' + hash; - var tx; - - this.db.get(id, function(err, data) { - if (err) { - if (err.type === 'NotFoundError') - return callback(); - return callback(err); - } - - try { - tx = bcoin.tx.fromExtended(data); - } catch (e) { - return callback(e); - } - - if (self.options.paranoid && tx.hash('hex') !== hash) - return callback(new Error('BlockDB is corrupt. All is lost.')); - - return callback(null, tx); - }); -}; - -BlockDB.prototype.getFullTX = function getFullTX(hash, callback) { - var self = this; - - return this.getTX(hash, function(err, tx) { - if (err) - return callback(err); - - if (!tx) - return callback(); - - return self.fillTX(tx, function(err) { - if (err) - return callback(err); - - return callback(null, tx); - }); - }); -}; - -BlockDB.prototype.getFullBlock = function getFullBlock(hash, callback) { - var self = this; - - return this.getBlock(hash, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(); - - return self.fillTX(block.txs, function(err) { - if (err) - return callback(err); - - return callback(null, block); - }); - }); -}; - -BlockDB.prototype._getCoinBlock = function _getCoinBlock(hash, callback) { - var self = this; - - if (hash instanceof bcoin.block) - return callback(null, hash); - - return this.getBlock(hash, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(); - - return self.fillBlock(block, callback); - }); -}; - -BlockDB.prototype._getTXBlock = function _getTXBlock(hash, callback) { - var self = this; - - if (hash instanceof bcoin.block) - return callback(null, hash); - - return this.getBlock(hash, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(); - - return self.fillTXBlock(block, callback); - }); -}; - -BlockDB.prototype.fillBlock = function fillBlock(block, callback) { - var self = this; - - return this.fillCoin(block.txs, function(err) { - var coins, i, tx, hash, j, input, id; - - if (err) - return callback(err); - - coins = {}; - - for (i = 0; i < block.txs.length; i++) { - tx = block.txs[i]; - hash = tx.hash('hex'); - - for (j = 0; j < tx.inputs.length; j++) { - input = tx.inputs[j]; - id = input.prevout.hash + '/' + input.prevout.index; - if (!input.output && coins[id]) { - input.output = coins[id]; - delete coins[id]; - } - } - - for (j = 0; j < tx.outputs.length; j++) - coins[hash + '/' + j] = bcoin.coin(tx, j); - } - - return callback(null, block); - }); -}; - -BlockDB.prototype.fillTXBlock = function fillTXBlock(block, callback) { - var self = this; - - return this.fillTX(block.txs, function(err) { - var coins, i, tx, hash, j, input, id; - - if (err) - return callback(err); - - coins = {}; - - for (i = 0; i < block.txs.length; i++) { - tx = block.txs[i]; - hash = tx.hash('hex'); - - for (j = 0; j < tx.inputs.length; j++) { - input = tx.inputs[j]; - id = input.prevout.hash + '/' + input.prevout.index; - if (!input.output && coins[id]) { - input.output = coins[id]; - delete coins[id]; - } - } - - for (j = 0; j < tx.outputs.length; j++) - coins[hash + '/' + j] = bcoin.coin(tx, j); - } - - return callback(null, block); - }); -}; - -BlockDB.prototype._getHash = function _getHash(height, callback) { - if (typeof height === 'string') - return callback(null, height); - - this.db.get('c/h/' + pad32(height), function(err, hash) { - if (err && err.type !== 'NotFoundError') - return callback(err); - if (!hash) - return callback(); - return callback(null, utils.toHex(hash)); - }); -}; - -BlockDB.prototype.getBlock = function getBlock(hash, callback) { - var self = this; - var id, block; - - return this._getHash(hash, function(err, hash) { - if (err) - return callback(err); - - if (!hash) - return callback(); - - id = 'b/b/' + hash; - - self.db.get(id, function(err, data) { - if (err) { - if (err.type === 'NotFoundError') - return callback(); - return callback(err); - } - - try { - block = bcoin.block.fromCompact(data); - } catch (e) { - return callback(e); - } - - block.txs = []; - - utils.forEach(block.hashes, function(hash, next, i) { - self.getTX(hash, function(err, tx) { - if (err) - return next(err); - - if (!tx) - return next(new Error('TX not found.')); - - block.txs[i] = tx; - - next(); - }); - }, function(err) { - if (err) - return callback(err); - - delete block.hashes; - block = new bcoin.block(block); - return callback(null, block); - }); - }); - }); -}; - -BlockDB.prototype.hasBlock = function hasBlock(hash, callback) { - var self = this; - var id = 'b/b/' + hash; - - if (typeof hash === 'number') - id = 'b/h/' + pad32(hash); - - this.db.get(id, function(err, data) { - if (err && err.type !== 'NotFoundError') - return callback(err); - - if (!data) - return callback(null, false); - - return callback(null, true); - }); -}; - -BlockDB.prototype.hasCoin = function hasCoin(hash, index, callback) { - var self = this; - var id = 'u/t/' + hash + '/' + index; - - this.db.get(id, function(err, data) { - if (err && err.type !== 'NotFoundError') - return callback(err); - - if (!data) - return callback(null, false); - - return callback(null, true); - }); -}; - -BlockDB.prototype._getTX = function _getTX(hash, callback) { - if (hash instanceof bcoin.tx) - return callback(null, hash); - return this.getTX(hash); -}; - -// For BIP30 -// https://bitcointalk.org/index.php?topic=67738.0 -BlockDB.prototype.isUnspentTX = function isUnspentTX(hash, callback) { - return this.isSpentTX(hash, function(err, spent) { - if (err) - return callback(err); - - return callback(null, !spent); - }); -}; - -BlockDB.prototype.isSpentTX = function isSpentTX(hash, callback) { - var spent = true; - - // Important! - if (hash.hash) - hash = hash.hash('hex'); - - var iter = this.db.db.iterator({ - gte: 'u/t/' + hash, - lte: 'u/t/' + hash + '~', - keys: true, - values: false, - fillCache: false, - keyAsBuffer: false - }); - - (function next() { - iter.next(function(err, key, value) { - if (err) { - return iter.end(function() { - done(err); - }); - } - - if (key === undefined) - return iter.end(done); - - spent = false; - - // IMPORTANT! - iter.end(done); - }); - })(); - - function done(err) { - if (err) - return callback(err); - - return callback(null, spent); - } -}; - -BlockDB.prototype.hasTX = function hasTX(hash, callback) { - var self = this; - var id = 't/t/' + hash; - - this.db.get(id, function(err, data) { - if (err && err.type !== 'NotFoundError') - return callback(err); - - if (!data) - return callback(null, false); - - return callback(null, true); - }); -}; - -BlockDB.prototype.isSpent = function isSpent(hash, index, callback) { - return this.hasCoin(hash, index, function(err, result) { - if (err) - return callback(err); - - return callback(null, !result); - }); -}; - -BlockDB.prototype._pruneBlock = function _pruneBlock(block, batch, callback) { - var self = this; - - // For much more aggressive pruning, we could delete - // the block headers 288 blocks before this one here as well. - - return utils.forEachSerial(block.txs, function(tx, next) { - self._pruneTX(tx, batch, next); - }, function(err) { - if (err) - return callback(err); - - self._pruneQueue(block, batch, function(err) { - if (err) - return callback(err); - return callback(); - }); - }); -}; - -BlockDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) { - var self = this; - var watermark = tx.height - self.keepBlocks; - - if (tx.isCoinbase()) - return callback(); - - utils.forEachSerial(tx.inputs, function(input, next) { - self.isSpentTX(input.prevout.hash, function(err, result) { - var futureHeight; - - if (err) - return next(err); - - if (!result) - return next(); - - // Output's tx is below the watermark. It's not - // safe to delete yet. Queue it up to be deleted - // at a future height. - if (watermark >= 0 && input.output.height > watermark) { - futureHeight = input.output.height + watermark; - // This may screw up txs that end up being - // in a side chain in the future, but technically - // they should be safe to delete anyway at the - // future height. It's unlikely there will be - // _another_ reorg to take over 288 blocks. - batch.put('t/q/' + futureHeight + '/' + input.prevout.hash, DUMMY); - return next(); - } - - self._removeTX(input.prevout.hash, batch, next); - }); - }, callback); -}; - -BlockDB.prototype._removeTX = function _removeTX(hash, batch, callback) { - var self = this; - var uniq = {}; - - batch.del('t/t/' + hash); - - if (!self.options.indexAddress) - return callback(); - - this.getTX(hash, function(err, tx) { - if (err) - return callback(err); - - if (!tx) - return callback(); - - tx.inputs.forEach(function(input) { - var address; - - if (input.isCoinbase()) - return; - - // Since we may have pruned these outputs, we have to - // guess at the address. Should be correct 90% of the - // time, though we may leave some fluff behind. Not - // a perfect pruning, but probably good enough. - address = input.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.del('t/a/' + address + '/' + hash); - } - }); - - tx.outputs.forEach(function(output, i) { - var address = output.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.del('t/a/' + address + '/' + hash); - } - }); - - callback(); - }); -}; - -BlockDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) { - var self = this; - var hashes = []; - var iter = self.db.db.iterator({ - gte: 't/q/' + block.height, - lte: 't/q/' + block.height + '~', - keys: true, - values: false, - fillCache: false, - keyAsBuffer: false - }); - - (function next() { - iter.next(function(err, key, value) { - var parts, hash, index; - - if (err) { - return iter.end(function() { - done(err); - }); - } - - if (key === undefined) - return iter.end(done); - - parts = key.split('/'); - hash = parts[3]; - - hashes.push(hash); - - next(); - }); - })(); - - function done(err) { - if (err) - return callback(err); - - if (hashes.length) - utils.debug('Retroactively pruning txs at height %d', block.height); - - utils.forEachSerial(hashes, function(hash, next) { - batch.del('t/q/' + block.height + '/' + hash); - self._removeTX(hash, batch, next); - }, callback); - } -}; - -BlockDB.prototype.batch = function batch() { - if (this.fsync) - return new utils.SyncBatch(this.db); - return this.db.batch(); -}; - -/** - * Expose - */ - -module.exports = BlockDB; diff --git a/lib/bcoin/fullnode.js b/lib/bcoin/fullnode.js index 8382f95d..49f5cf40 100644 --- a/lib/bcoin/fullnode.js +++ b/lib/bcoin/fullnode.js @@ -96,16 +96,14 @@ Fullnode.prototype._init = function _init() { self.emit('error', err); }); - if (0) - this.on('tx', function(tx) { - self.walletdb.addTX(tx, function(err) { - if (err) - self.emit('error', err); - }); - }); + // this.on('tx', function(tx) { + // self.walletdb.addTX(tx, function(err) { + // if (err) + // self.emit('error', err); + // }); + // }); // Emit events for valid blocks and TXs. - if (0) this.chain.on('block', function(block) { self.emit('block', block); block.txs.forEach(function(tx) { @@ -118,16 +116,14 @@ Fullnode.prototype._init = function _init() { }); // Update the mempool. - if (0) - this.chain.on('add block', function(block) { - self.mempool.addBlock(block); - }); + // this.chain.on('add block', function(block) { + // self.mempool.addBlock(block); + // }); - if (0) - this.chain.on('remove block', function(block) { - self.mempool.removeBlock(block); - self.walletdb.removeBlock(block); - }); + // this.chain.on('remove block', function(block) { + // self.mempool.removeBlock(block); + // self.walletdb.removeBlock(block); + // }); function load() { if (!--pending) { diff --git a/lib/bcoin/node.js b/lib/bcoin/node.js index 34afb643..ec24b4d4 100644 --- a/lib/bcoin/node.js +++ b/lib/bcoin/node.js @@ -38,7 +38,6 @@ function Node(options) { network.set(this.options.network); this.network = network; - this.blockdb = null; this.mempool = null; this.pool = null; this.chain = null; diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index 8fca95cc..dafe7a7e 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -51,7 +51,6 @@ function Pool(node, options) { this.size = options.size || 8; this.chain = node.chain; - this.blockdb = node.blockdb; this.mempool = node.mempool; if (options.spv) { @@ -1539,9 +1538,9 @@ Pool.prototype._sendRequests = function _sendRequests(peer) { else if (this.chain.height <= 150000) size = 250; else if (this.chain.height <= 170000) - size = this.blockdb ? 4 : 40; + size = 20; else - size = this.blockdb ? 1 : 10; + size = 10; items = peer.queue.block.slice(0, size); peer.queue.block = peer.queue.block.slice(size);