better pruning. use datastore only in node.
This commit is contained in:
Christopher Jeffrey
parent
bb306c3547
commit
f4a1ca3eaf
@ -14,7 +14,6 @@ var utils = bcoin.utils;
|
||||
var assert = utils.assert;
|
||||
var pad32 = utils.pad32;
|
||||
var DUMMY = new Buffer([]);
|
||||
var DataStore = require('./datastore');
|
||||
|
||||
/**
|
||||
* ChainDB
|
||||
@ -75,7 +74,10 @@ ChainDB.prototype._init = function _init() {
|
||||
cacheSize: 16 * 1024 * 1024,
|
||||
writeBufferSize: 8 * 1024 * 1024
|
||||
});
|
||||
this.db = new DataStore(this.db);
|
||||
if (!bcoin.isBrowser) {
|
||||
var DataStore = require('./data' + 'store');
|
||||
this.db = new DataStore(this.db);
|
||||
}
|
||||
};
|
||||
|
||||
ChainDB.prototype.load = function load(callback) {
|
||||
@ -373,20 +375,10 @@ ChainDB.prototype.save = function save(entry, block, callback) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
|
||||
return batch.write(function(err) {
|
||||
self._pruneBlock(block, batch, function(err) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
|
||||
// We have to do this as a separate
|
||||
// batch because of the isUnspentTX call.
|
||||
// Not ideal, but it won't break anything
|
||||
// if there is crash inconsistency. Just a
|
||||
// less-than-perfect pruning would result.
|
||||
self._pruneBlock(block, function(err) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
return callback();
|
||||
});
|
||||
return batch.write(callback);
|
||||
});
|
||||
});
|
||||
};
|
||||
@ -627,7 +619,7 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
|
||||
if (self.options.indexAddress) {
|
||||
address = input.getAddress();
|
||||
|
||||
if (address && !uniq[address]) {
|
||||
if (address && !uniq[address] && !self.prune) {
|
||||
uniq[address] = true;
|
||||
batch.put('t/a/' + address + '/' + hash, DUMMY);
|
||||
}
|
||||
@ -649,7 +641,7 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
|
||||
if (self.options.indexAddress) {
|
||||
address = output.getAddress();
|
||||
|
||||
if (address && !uniq[address]) {
|
||||
if (address && !uniq[address] && !self.prune) {
|
||||
uniq[address] = true;
|
||||
batch.put('t/a/' + address + '/' + hash, DUMMY);
|
||||
}
|
||||
@ -699,7 +691,7 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callba
|
||||
if (self.options.indexAddress) {
|
||||
address = input.getAddress();
|
||||
|
||||
if (address && !uniq[address]) {
|
||||
if (address && !uniq[address] && !self.prune) {
|
||||
uniq[address] = true;
|
||||
batch.del('t/a/' + address + '/' + hash);
|
||||
}
|
||||
@ -724,7 +716,7 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callba
|
||||
if (self.options.indexAddress) {
|
||||
address = output.getAddress();
|
||||
|
||||
if (address && !uniq[address]) {
|
||||
if (address && !uniq[address] && !self.prune) {
|
||||
uniq[address] = true;
|
||||
batch.del('t/a/' + address + '/' + hash);
|
||||
}
|
||||
@ -799,6 +791,27 @@ ChainDB.prototype.fillTX = function fillTX(tx, callback) {
|
||||
if (tx.isCoinbase())
|
||||
return callback(null, tx);
|
||||
|
||||
if (self.prune) {
|
||||
return utils.forEachSerial(tx.inputs, function(input, next) {
|
||||
if (input.output)
|
||||
return next();
|
||||
|
||||
self._getPruneCoin(input.prevout.hash, input.prevout.index, function(err, coin) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
|
||||
if (coin)
|
||||
input.output = coin;
|
||||
|
||||
next();
|
||||
});
|
||||
}, function(err) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
return callback(null, tx);
|
||||
});
|
||||
}
|
||||
|
||||
utils.forEachSerial(tx.inputs, function(input, next) {
|
||||
if (input.output)
|
||||
return next();
|
||||
@ -1147,24 +1160,11 @@ ChainDB.prototype.fillTXBlock = function fillTXBlock(block, callback) {
|
||||
});
|
||||
};
|
||||
|
||||
ChainDB.prototype._getHash = function _getHash(height, callback) {
|
||||
if (typeof height === 'string')
|
||||
return callback(null, height);
|
||||
|
||||
this.db.get('c/h/' + pad32(height), function(err, hash) {
|
||||
if (err && err.type !== 'NotFoundError')
|
||||
return callback(err);
|
||||
if (!hash)
|
||||
return callback();
|
||||
return callback(null, utils.toHex(hash));
|
||||
});
|
||||
};
|
||||
|
||||
ChainDB.prototype.getBlock = function getBlock(hash, callback) {
|
||||
var self = this;
|
||||
var id, block;
|
||||
|
||||
return this._getHash(hash, function(err, hash) {
|
||||
return this.getHash(hash, function(err, hash) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
|
||||
@ -1323,9 +1323,9 @@ ChainDB.prototype.isSpent = function isSpent(hash, index, callback) {
|
||||
});
|
||||
};
|
||||
|
||||
ChainDB.prototype._pruneBlock = function _pruneBlock(block, callback) {
|
||||
ChainDB.prototype._pruneBlock = function _pruneBlock(block, batch, callback) {
|
||||
var self = this;
|
||||
var batch;
|
||||
var futureHeight;
|
||||
|
||||
if (self.options.spv)
|
||||
return callback();
|
||||
@ -1333,137 +1333,80 @@ ChainDB.prototype._pruneBlock = function _pruneBlock(block, callback) {
|
||||
if (!self.prune)
|
||||
return callback();
|
||||
|
||||
batch = self.batch();
|
||||
|
||||
// For much more aggressive pruning, we could delete
|
||||
// the block headers 288 blocks before this one here as well.
|
||||
batch.put('b/q/' + pad32(block.height + self.keepBlocks), block.hash());
|
||||
|
||||
return utils.forEachSerial(block.txs, function(tx, next) {
|
||||
self._pruneTX(tx, batch, next);
|
||||
}, function(err) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
|
||||
self._pruneQueue(block, batch, function(err) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
return batch.write(callback);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) {
|
||||
var self = this;
|
||||
var watermark = tx.height - self.keepBlocks;
|
||||
|
||||
if (tx.isCoinbase())
|
||||
// Keep the genesis block
|
||||
if (block.isGenesis())
|
||||
return callback();
|
||||
|
||||
utils.forEachSerial(tx.inputs, function(input, next) {
|
||||
self.isSpentTX(input.prevout.hash, function(err, result) {
|
||||
var futureHeight;
|
||||
futureHeight = pad32(block.height + self.keepBlocks);
|
||||
|
||||
if (err)
|
||||
return next(err);
|
||||
|
||||
if (!result)
|
||||
return next();
|
||||
|
||||
// Output's tx is below the watermark. It's not
|
||||
// safe to delete yet. Queue it up to be deleted
|
||||
// at a future height.
|
||||
if (watermark >= 0 && input.output.height > watermark) {
|
||||
futureHeight = input.output.height + watermark;
|
||||
// This may screw up txs that end up being
|
||||
// in a side chain in the future, but technically
|
||||
// they should be safe to delete anyway at the
|
||||
// future height. It's unlikely there will be
|
||||
// _another_ reorg to take over 288 blocks.
|
||||
batch.put(
|
||||
't/q/' + pad32(futureHeight) + '/' + input.prevout.hash,
|
||||
DUMMY);
|
||||
return next();
|
||||
}
|
||||
|
||||
self._removeTX(input.prevout.hash, batch, next);
|
||||
});
|
||||
}, callback);
|
||||
};
|
||||
|
||||
ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) {
|
||||
var self = this;
|
||||
|
||||
batch.put(
|
||||
't/q/' + pad32(tx.height + self.keepBlocks) + '/' + tx.hash('hex'),
|
||||
DUMMY);
|
||||
return callback();
|
||||
|
||||
if (tx.isCoinbase())
|
||||
return callback();
|
||||
|
||||
tx.inputs.forEach(function(input) {
|
||||
batch.put(
|
||||
't/q/' + pad32(tx.height + self.keepBlocks) + '/' + input.prevout.hash,
|
||||
DUMMY);
|
||||
});
|
||||
|
||||
callback();
|
||||
};
|
||||
|
||||
ChainDB.prototype._removeTX = function _removeTX(hash, batch, callback) {
|
||||
var self = this;
|
||||
var uniq = {};
|
||||
|
||||
batch.del('t/t/' + hash);
|
||||
|
||||
if (!self.options.indexAddress)
|
||||
return callback();
|
||||
|
||||
this.getTX(hash, function(err, tx) {
|
||||
if (err)
|
||||
return callback(err);
|
||||
|
||||
if (!tx)
|
||||
return callback();
|
||||
batch.put('b/q/' + futureHeight, block.hash());
|
||||
|
||||
block.txs.forEach(function(tx, i) {
|
||||
tx.inputs.forEach(function(input) {
|
||||
var address;
|
||||
|
||||
if (input.isCoinbase())
|
||||
return;
|
||||
|
||||
// Since we may have pruned these outputs, we have to
|
||||
// guess at the address. Should be correct 90% of the
|
||||
// time, though we may leave some fluff behind. Not
|
||||
// a perfect pruning, but probably good enough.
|
||||
address = input.getAddress();
|
||||
assert(input.output);
|
||||
|
||||
if (address && !uniq[address]) {
|
||||
uniq[address] = true;
|
||||
batch.del('t/a/' + address + '/' + hash);
|
||||
}
|
||||
batch.put('u/x/'
|
||||
+ input.prevout.hash
|
||||
+ '/' + input.prevout.index,
|
||||
input.output.toExtended());
|
||||
|
||||
batch.put('u/q/'
|
||||
+ futureHeight
|
||||
+ '/' + input.prevout.hash
|
||||
+ '/' + input.prevout.index,
|
||||
DUMMY);
|
||||
});
|
||||
|
||||
tx.outputs.forEach(function(output, i) {
|
||||
var address = output.getAddress();
|
||||
|
||||
if (address && !uniq[address]) {
|
||||
uniq[address] = true;
|
||||
batch.del('t/a/' + address + '/' + hash);
|
||||
}
|
||||
});
|
||||
|
||||
callback();
|
||||
});
|
||||
|
||||
self._pruneQueue(block, batch, callback);
|
||||
};
|
||||
|
||||
ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) {
|
||||
var self = this;
|
||||
var hashes = [];
|
||||
var key = 'b/q/' + pad32(block.height);
|
||||
self.db.get(key, function(err, hash) {
|
||||
if (err && err.type !== 'NotFoundError')
|
||||
return callback(err);
|
||||
|
||||
if (!hash)
|
||||
return callback();
|
||||
|
||||
hash = utils.toHex(hash);
|
||||
|
||||
self.db.get('b/b/' + hash, function(err, cblock) {
|
||||
if (err && err.type !== 'NotFoundError')
|
||||
return callback(err);
|
||||
|
||||
batch.del(key);
|
||||
|
||||
if (!cblock)
|
||||
return callback();
|
||||
|
||||
try {
|
||||
cblock = bcoin.block.fromCompact(cblock);
|
||||
} catch (e) {
|
||||
return callback(e);
|
||||
}
|
||||
|
||||
batch.del('b/b/' + hash);
|
||||
|
||||
cblock.hashes.forEach(function(hash) {
|
||||
batch.del('t/t/' + hash);
|
||||
});
|
||||
|
||||
self._pruneCoinQueue(block, batch, callback);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
ChainDB.prototype._pruneCoinQueue = function _pruneQueue(block, batch, callback) {
|
||||
var self = this;
|
||||
var iter = self.db.db.iterator({
|
||||
gte: 't/q/' + pad32(block.height),
|
||||
lte: 't/q/' + pad32(block.height) + '~',
|
||||
gte: 'u/q/' + pad32(block.height),
|
||||
lte: 'u/q/' + pad32(block.height) + '~',
|
||||
keys: true,
|
||||
values: false,
|
||||
fillCache: false,
|
||||
@ -1472,48 +1415,49 @@ ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) {
|
||||
|
||||
(function next() {
|
||||
iter.next(function(err, key, value) {
|
||||
var parts, hash;
|
||||
var parts, hash, index;
|
||||
|
||||
if (err) {
|
||||
return iter.end(function() {
|
||||
done(err);
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
if (key === undefined)
|
||||
return iter.end(done);
|
||||
return iter.end(callback);
|
||||
|
||||
parts = key.split('/');
|
||||
hash = parts[3];
|
||||
index = +parts[4];
|
||||
|
||||
hashes.push(hash);
|
||||
batch.del(key);
|
||||
batch.del('u/x/' + hash + '/' + index);
|
||||
|
||||
next();
|
||||
});
|
||||
})();
|
||||
};
|
||||
|
||||
function done(err) {
|
||||
if (err)
|
||||
ChainDB.prototype._getPruneCoin = function _getPruneCoin(hash, index, callback) {
|
||||
var self = this;
|
||||
var id = 'u/x/' + hash + '/' + index;
|
||||
var coin;
|
||||
|
||||
this.db.get(id, function(err, data) {
|
||||
if (err && err.type !== 'NotFoundError')
|
||||
return callback(err);
|
||||
|
||||
// if (hashes.length)
|
||||
// utils.debug('Retroactively pruning txs at height %d.', block.height);
|
||||
if (!data)
|
||||
return self.getCoin(hash, index, callback);
|
||||
|
||||
self.db.get('b/q/' + pad32(block.height), function(err, hash) {
|
||||
if (err && err.type !== 'NotFoundError')
|
||||
return callback(err);
|
||||
try {
|
||||
coin = bcoin.coin.fromExtended(data);
|
||||
} catch (e) {
|
||||
return callback(e);
|
||||
}
|
||||
|
||||
if (hash) {
|
||||
batch.del('b/q/' + pad32(block.height));
|
||||
batch.del('b/b/' + utils.toHex(hash));
|
||||
}
|
||||
|
||||
utils.forEachSerial(hashes, function(hash, next) {
|
||||
batch.del('t/q/' + pad32(block.height) + '/' + hash);
|
||||
self._removeTX(hash, batch, next);
|
||||
}, callback);
|
||||
});
|
||||
}
|
||||
return callback(null, coin);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@ -15,8 +15,7 @@ var fs = bcoin.fs;
|
||||
var pad32 = utils.pad32;
|
||||
|
||||
var MAX_FILE_SIZE = 512 * 1024 * 1024;
|
||||
var MAX_FILE_SIZE = 1 * 1024 * 1024;
|
||||
var MAX_FILE_SIZE = 100 * 1024;
|
||||
var MAX_FILE_SIZE = 10 * 1024 * 1024;
|
||||
var NULL_CHUNK = new Buffer([0xff, 0xff, 0xff, 0xff]);
|
||||
|
||||
/**
|
||||
@ -488,9 +487,6 @@ DataStore.prototype.delData = function delData(off, callback, force) {
|
||||
if (offset + size + 12 !== fsize)
|
||||
return callback();
|
||||
|
||||
utils.debug('Pruning block files...');
|
||||
utils.debug({ offset: offset, size: size, total: offset + size, fsize: fsize });
|
||||
|
||||
// If we're deleting the last record, traverse
|
||||
// through the reverse linked list of undo offsets
|
||||
// until we hit a record that isn't deleted.
|
||||
@ -512,7 +508,6 @@ DataStore.prototype.delData = function delData(off, callback, force) {
|
||||
|
||||
function done() {
|
||||
// Delete the file if nothing is in it.
|
||||
utils.debug('Truncating to %d', offset);
|
||||
if (offset === 0) {
|
||||
self.pool.remove(index);
|
||||
return fs.unlink(self.dir + '/f' + pad32(index), callback);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user