diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index 2788de2b..b478b57b 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -14,7 +14,6 @@ var utils = bcoin.utils; var assert = utils.assert; var pad32 = utils.pad32; var DUMMY = new Buffer([]); -var DataStore = require('./datastore'); /** * ChainDB @@ -75,7 +74,10 @@ ChainDB.prototype._init = function _init() { cacheSize: 16 * 1024 * 1024, writeBufferSize: 8 * 1024 * 1024 }); - this.db = new DataStore(this.db); + if (!bcoin.isBrowser) { + var DataStore = require('./data' + 'store'); + this.db = new DataStore(this.db); + } }; ChainDB.prototype.load = function load(callback) { @@ -373,20 +375,10 @@ ChainDB.prototype.save = function save(entry, block, callback) { if (err) return callback(err); - return batch.write(function(err) { + self._pruneBlock(block, batch, function(err) { if (err) return callback(err); - - // We have to do this as a separate - // batch because of the isUnspentTX call. - // Not ideal, but it won't break anything - // if there is crash inconsistency. Just a - // less-than-perfect pruning would result. - self._pruneBlock(block, function(err) { - if (err) - return callback(err); - return callback(); - }); + return batch.write(callback); }); }); }; @@ -627,7 +619,7 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) { if (self.options.indexAddress) { address = input.getAddress(); - if (address && !uniq[address]) { + if (address && !uniq[address] && !self.prune) { uniq[address] = true; batch.put('t/a/' + address + '/' + hash, DUMMY); } @@ -649,7 +641,7 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) { if (self.options.indexAddress) { address = output.getAddress(); - if (address && !uniq[address]) { + if (address && !uniq[address] && !self.prune) { uniq[address] = true; batch.put('t/a/' + address + '/' + hash, DUMMY); } @@ -699,7 +691,7 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callba if (self.options.indexAddress) { address = input.getAddress(); - if (address && !uniq[address]) { + if (address && !uniq[address] && !self.prune) { uniq[address] = true; batch.del('t/a/' + address + '/' + hash); } @@ -724,7 +716,7 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callba if (self.options.indexAddress) { address = output.getAddress(); - if (address && !uniq[address]) { + if (address && !uniq[address] && !self.prune) { uniq[address] = true; batch.del('t/a/' + address + '/' + hash); } @@ -799,6 +791,27 @@ ChainDB.prototype.fillTX = function fillTX(tx, callback) { if (tx.isCoinbase()) return callback(null, tx); + if (self.prune) { + return utils.forEachSerial(tx.inputs, function(input, next) { + if (input.output) + return next(); + + self._getPruneCoin(input.prevout.hash, input.prevout.index, function(err, coin) { + if (err) + return callback(err); + + if (coin) + input.output = coin; + + next(); + }); + }, function(err) { + if (err) + return callback(err); + return callback(null, tx); + }); + } + utils.forEachSerial(tx.inputs, function(input, next) { if (input.output) return next(); @@ -1147,24 +1160,11 @@ ChainDB.prototype.fillTXBlock = function fillTXBlock(block, callback) { }); }; -ChainDB.prototype._getHash = function _getHash(height, callback) { - if (typeof height === 'string') - return callback(null, height); - - this.db.get('c/h/' + pad32(height), function(err, hash) { - if (err && err.type !== 'NotFoundError') - return callback(err); - if (!hash) - return callback(); - return callback(null, utils.toHex(hash)); - }); -}; - ChainDB.prototype.getBlock = function getBlock(hash, callback) { var self = this; var id, block; - return this._getHash(hash, function(err, hash) { + return this.getHash(hash, function(err, hash) { if (err) return callback(err); @@ -1323,9 +1323,9 @@ ChainDB.prototype.isSpent = function isSpent(hash, index, callback) { }); }; -ChainDB.prototype._pruneBlock = function _pruneBlock(block, callback) { +ChainDB.prototype._pruneBlock = function _pruneBlock(block, batch, callback) { var self = this; - var batch; + var futureHeight; if (self.options.spv) return callback(); @@ -1333,137 +1333,80 @@ ChainDB.prototype._pruneBlock = function _pruneBlock(block, callback) { if (!self.prune) return callback(); - batch = self.batch(); - - // For much more aggressive pruning, we could delete - // the block headers 288 blocks before this one here as well. - batch.put('b/q/' + pad32(block.height + self.keepBlocks), block.hash()); - - return utils.forEachSerial(block.txs, function(tx, next) { - self._pruneTX(tx, batch, next); - }, function(err) { - if (err) - return callback(err); - - self._pruneQueue(block, batch, function(err) { - if (err) - return callback(err); - return batch.write(callback); - }); - }); -}; - -ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) { - var self = this; - var watermark = tx.height - self.keepBlocks; - - if (tx.isCoinbase()) + // Keep the genesis block + if (block.isGenesis()) return callback(); - utils.forEachSerial(tx.inputs, function(input, next) { - self.isSpentTX(input.prevout.hash, function(err, result) { - var futureHeight; + futureHeight = pad32(block.height + self.keepBlocks); - if (err) - return next(err); - - if (!result) - return next(); - - // Output's tx is below the watermark. It's not - // safe to delete yet. Queue it up to be deleted - // at a future height. - if (watermark >= 0 && input.output.height > watermark) { - futureHeight = input.output.height + watermark; - // This may screw up txs that end up being - // in a side chain in the future, but technically - // they should be safe to delete anyway at the - // future height. It's unlikely there will be - // _another_ reorg to take over 288 blocks. - batch.put( - 't/q/' + pad32(futureHeight) + '/' + input.prevout.hash, - DUMMY); - return next(); - } - - self._removeTX(input.prevout.hash, batch, next); - }); - }, callback); -}; - -ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) { - var self = this; - - batch.put( - 't/q/' + pad32(tx.height + self.keepBlocks) + '/' + tx.hash('hex'), - DUMMY); - return callback(); - - if (tx.isCoinbase()) - return callback(); - - tx.inputs.forEach(function(input) { - batch.put( - 't/q/' + pad32(tx.height + self.keepBlocks) + '/' + input.prevout.hash, - DUMMY); - }); - - callback(); -}; - -ChainDB.prototype._removeTX = function _removeTX(hash, batch, callback) { - var self = this; - var uniq = {}; - - batch.del('t/t/' + hash); - - if (!self.options.indexAddress) - return callback(); - - this.getTX(hash, function(err, tx) { - if (err) - return callback(err); - - if (!tx) - return callback(); + batch.put('b/q/' + futureHeight, block.hash()); + block.txs.forEach(function(tx, i) { tx.inputs.forEach(function(input) { - var address; - if (input.isCoinbase()) return; - // Since we may have pruned these outputs, we have to - // guess at the address. Should be correct 90% of the - // time, though we may leave some fluff behind. Not - // a perfect pruning, but probably good enough. - address = input.getAddress(); + assert(input.output); - if (address && !uniq[address]) { - uniq[address] = true; - batch.del('t/a/' + address + '/' + hash); - } + batch.put('u/x/' + + input.prevout.hash + + '/' + input.prevout.index, + input.output.toExtended()); + + batch.put('u/q/' + + futureHeight + + '/' + input.prevout.hash + + '/' + input.prevout.index, + DUMMY); }); - - tx.outputs.forEach(function(output, i) { - var address = output.getAddress(); - - if (address && !uniq[address]) { - uniq[address] = true; - batch.del('t/a/' + address + '/' + hash); - } - }); - - callback(); }); + + self._pruneQueue(block, batch, callback); }; ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) { var self = this; - var hashes = []; + var key = 'b/q/' + pad32(block.height); + self.db.get(key, function(err, hash) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!hash) + return callback(); + + hash = utils.toHex(hash); + + self.db.get('b/b/' + hash, function(err, cblock) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + batch.del(key); + + if (!cblock) + return callback(); + + try { + cblock = bcoin.block.fromCompact(cblock); + } catch (e) { + return callback(e); + } + + batch.del('b/b/' + hash); + + cblock.hashes.forEach(function(hash) { + batch.del('t/t/' + hash); + }); + + self._pruneCoinQueue(block, batch, callback); + }); + }); +}; + +ChainDB.prototype._pruneCoinQueue = function _pruneQueue(block, batch, callback) { + var self = this; var iter = self.db.db.iterator({ - gte: 't/q/' + pad32(block.height), - lte: 't/q/' + pad32(block.height) + '~', + gte: 'u/q/' + pad32(block.height), + lte: 'u/q/' + pad32(block.height) + '~', keys: true, values: false, fillCache: false, @@ -1472,48 +1415,49 @@ ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) { (function next() { iter.next(function(err, key, value) { - var parts, hash; + var parts, hash, index; if (err) { return iter.end(function() { - done(err); + callback(err); }); } if (key === undefined) - return iter.end(done); + return iter.end(callback); parts = key.split('/'); hash = parts[3]; + index = +parts[4]; - hashes.push(hash); + batch.del(key); + batch.del('u/x/' + hash + '/' + index); next(); }); })(); +}; - function done(err) { - if (err) +ChainDB.prototype._getPruneCoin = function _getPruneCoin(hash, index, callback) { + var self = this; + var id = 'u/x/' + hash + '/' + index; + var coin; + + this.db.get(id, function(err, data) { + if (err && err.type !== 'NotFoundError') return callback(err); - // if (hashes.length) - // utils.debug('Retroactively pruning txs at height %d.', block.height); + if (!data) + return self.getCoin(hash, index, callback); - self.db.get('b/q/' + pad32(block.height), function(err, hash) { - if (err && err.type !== 'NotFoundError') - return callback(err); + try { + coin = bcoin.coin.fromExtended(data); + } catch (e) { + return callback(e); + } - if (hash) { - batch.del('b/q/' + pad32(block.height)); - batch.del('b/b/' + utils.toHex(hash)); - } - - utils.forEachSerial(hashes, function(hash, next) { - batch.del('t/q/' + pad32(block.height) + '/' + hash); - self._removeTX(hash, batch, next); - }, callback); - }); - } + return callback(null, coin); + }); }; /** diff --git a/lib/bcoin/datastore.js b/lib/bcoin/datastore.js index d9c46609..12be9274 100644 --- a/lib/bcoin/datastore.js +++ b/lib/bcoin/datastore.js @@ -15,8 +15,7 @@ var fs = bcoin.fs; var pad32 = utils.pad32; var MAX_FILE_SIZE = 512 * 1024 * 1024; -var MAX_FILE_SIZE = 1 * 1024 * 1024; -var MAX_FILE_SIZE = 100 * 1024; +var MAX_FILE_SIZE = 10 * 1024 * 1024; var NULL_CHUNK = new Buffer([0xff, 0xff, 0xff, 0xff]); /** @@ -488,9 +487,6 @@ DataStore.prototype.delData = function delData(off, callback, force) { if (offset + size + 12 !== fsize) return callback(); - utils.debug('Pruning block files...'); - utils.debug({ offset: offset, size: size, total: offset + size, fsize: fsize }); - // If we're deleting the last record, traverse // through the reverse linked list of undo offsets // until we hit a record that isn't deleted. @@ -512,7 +508,6 @@ DataStore.prototype.delData = function delData(off, callback, force) { function done() { // Delete the file if nothing is in it. - utils.debug('Truncating to %d', offset); if (offset === 0) { self.pool.remove(index); return fs.unlink(self.dir + '/f' + pad32(index), callback);