move height lookups to chaindb.

This commit is contained in:
Christopher Jeffrey 2016-02-18 21:01:34 -08:00
parent d2a4f995fd
commit 5053fa2eeb
2 changed files with 199 additions and 249 deletions

View File

@ -35,7 +35,6 @@ function Chain(options) {
bcoin.debug = this.options.debug;
this.db = new bcoin.chaindb(this);
this.heightLookup = {};
this.request = new utils.RequestCache();
this.loading = false;
this.tip = null;
@ -145,6 +144,11 @@ Chain.prototype._init = function _init() {
self.mempool.removeBlock(block);
});
this.db.on('tip', function(tip) {
self.tip = tip;
self.height = tip.height;
});
this.loading = true;
utils.debug('Chain is loading.');
@ -154,56 +158,23 @@ Chain.prototype._init = function _init() {
throw err;
self._preload(function(err, start) {
var count = self.db.count();
var i = start || 1;
var lastEntry;
if (err) {
utils.debug('Preloading chain failed.');
utils.debug('Reason: %s', err.message);
}
utils.debug('Starting chain load at height: %s', i);
function done(height) {
if (height != null) {
utils.debug(
'Blockchain is corrupt after height %d. Resetting.',
height);
self.resetHeight(height);
} else {
utils.debug('Chain successfully loaded.');
}
self.db.load(start || 0, function(err) {
if (err)
throw err;
self.syncHeight(function(err) {
if (err)
throw err;
self.loading = false;
self.emit('load');
});
}
(function next() {
if (i >= count)
return done();
self.db.getAsync(i, function(err, entry) {
if (err)
throw err;
// Do some paranoid checks.
if (lastEntry && entry.prevBlock !== lastEntry.hash)
return done(Math.max(0, i - 2));
if (i % 10000 === 0)
utils.debug('Loaded %d blocks.', i);
lastEntry = entry;
self._saveEntry(entry);
i += 1;
next();
});
})();
});
});
});
};
@ -213,7 +184,7 @@ Chain.prototype._ensureGenesis = function _ensureGenesis(callback) {
callback = utils.asyncify(callback);
this._saveEntry(bcoin.chainblock.fromJSON(this, {
this.db.save(bcoin.chainblock.fromJSON(this, {
hash: network.genesis.hash,
version: network.genesis.version,
prevBlock: network.genesis.prevBlock,
@ -222,7 +193,7 @@ Chain.prototype._ensureGenesis = function _ensureGenesis(callback) {
bits: network.genesis.bits,
nonce: network.genesis.nonce,
height: 0
}), true);
}));
if (!this.blockdb)
return callback();
@ -348,11 +319,7 @@ Chain.prototype._preload = function _preload(callback) {
return;
}
// Don't write blocks we already have
// (bad for calculating chainwork).
// self._saveEntry(entry, height > chainHeight);
self._saveEntry(entry, true);
self.db.save(entry);
height++;
@ -686,8 +653,8 @@ Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
callback = utils.asyncify(callback);
// Already added
if (this.heightLookup[entry.hash] != null) {
assert(this.heightLookup[entry.hash] === entry.height);
if (this.db.has(entry.height)) {
assert(this.db.getHeight(entry.hash) === entry.height);
return callback(null, false);
}
@ -700,7 +667,7 @@ Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
if (err)
return callback(err);
self._saveEntry(entry, true, function(err) {
self.db.save(entry, function(err) {
if (err)
return callback(err);
@ -709,36 +676,6 @@ Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
});
};
Chain.prototype._saveEntry = function _saveEntry(entry, save, callback) {
this.heightLookup[entry.hash] = entry.height;
if (!this.tip || entry.height > this.tip.height) {
this.tip = entry;
this.height = this.tip.height;
this.emit('tip', this.tip);
}
if (save)
this.db.save(entry, callback);
};
Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) {
var heights = Object.keys(network.checkpoints).sort();
var index = heights.indexOf(height) - 1;
var checkpoint = network.checkpoint[index];
assert(index >= 0);
assert(checkpoint);
// This is the safest way to do it, the other
// possibility is to simply reset ignore the
// bad checkpoint block. The likelihood of
// someone carrying on an entire fork between
// to checkpoints is absurd, so this is
// probably _a lot_ of work for nothing.
this.resetHeight(checkpoint.height);
};
Chain.prototype.resetHeight = function resetHeight(height) {
var self = this;
var count = this.db.count();
@ -750,15 +687,7 @@ Chain.prototype.resetHeight = function resetHeight(height) {
if (height === count - 1)
return;
for (i = height + 1; i < count; i++) {
existing = this.db.get(i);
assert(existing);
// this.db.remove(i);
this.db.drop(i);
delete this.heightLookup[existing.hash];
}
this.db.truncate(height);
this.db.resetHeight(height);
// Reset the orphan map completely. There may
// have been some orphans on a forked chain we
@ -768,11 +697,6 @@ Chain.prototype.resetHeight = function resetHeight(height) {
this.orphan.bmap = {};
this.orphan.count = 0;
this.orphan.size = 0;
this.tip = this.db.get(height);
assert(this.tip);
this.height = this.tip.height;
this.emit('tip', this.tip);
};
Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
@ -789,26 +713,7 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
lock = this.locked;
this.locked = true;
i = height + 1;
function next() {
if (i === count)
return self.db.truncateAsync(height, done);
self.db.getAsync(i, function(err, existing) {
if (err)
return done(err);
assert(existing);
delete self.heightLookup[existing.hash];
self.db.drop(i);
i++;
next();
});
}
function done(err) {
this.db.resetHeightAsync(height, function(err) {
self.locked = lock;
if (err)
@ -823,13 +728,8 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
self.orphan.count = 0;
self.orphan.size = 0;
self.tip = self.db.get(height);
assert(self.tip);
self.height = self.tip.height;
self.emit('tip', self.tip);
return callback();
}
});
};
Chain.prototype.revertHeight = function revertHeight(height, callback) {
@ -1008,7 +908,7 @@ Chain.prototype.add = function add(initial, peer, callback) {
}
// Find the previous block height/index.
prevHeight = self.heightLookup[prevHash];
prevHeight = self.db.getHeight(prevHash);
// Validate the block we want to add.
// This is only necessary for new
@ -1140,7 +1040,7 @@ Chain.prototype.add = function add(initial, peer, callback) {
// don't store by hash so we can't compare
// chainworks. We reset the chain, find a
// new peer, and wait to see who wins.
assert(self.heightLookup[entry.hash] == null);
assert(self.db.getHeight(entry.hash) == null);
// The tip has more chainwork, it is a
// higher height than the entry. This is
@ -1168,7 +1068,7 @@ Chain.prototype.add = function add(initial, peer, callback) {
}
// Add entry if we do not have it.
assert(self.heightLookup[entry.hash] == null);
assert(self.db.getHeight(entry.hash) == null);
// Lookup previous entry.
prev = self.db.get(prevHeight);
@ -1314,7 +1214,7 @@ Chain.prototype.byHash = function byHash(hash) {
else if (hash.hash)
hash = hash.hash('hex');
return this.byHeight(this.heightLookup[hash]);
return this.byHeight(this.db.getHeight(hash));
};
Chain.prototype.byTime = function byTime(ts) {
@ -1425,7 +1325,7 @@ Chain.prototype.getLocator = function getLocator(start) {
}
if (typeof start === 'string') {
top = this.heightLookup[start];
top = this.db.getHeight(start);
if (top == null) {
// We could simply `return [start]` here,
// but there is no standardized "spacing"
@ -1474,7 +1374,7 @@ Chain.prototype.getLocatorAsync = function getLocatorAsync(start, callback) {
}
if (typeof start === 'string') {
top = this.heightLookup[start];
top = this.db.getHeight(start);
if (top == null) {
// We could simply `return [start]` here,
// but there is no standardized "spacing"
@ -1668,34 +1568,6 @@ Chain.prototype.retarget = function retarget(last, first) {
return utils.toCompact(target);
};
Chain.prototype.toJSON = function toJSON() {
var entries = [];
var count = this.db.count();
var i;
for (i = 0; i < count; i++)
entries.push(this.db.get(i));
return {
v: 2,
type: 'chain',
network: network.type,
entries: entries.map(function(entry) {
return entry.toJSON();
})
};
};
Chain.prototype.fromJSON = function fromJSON(json) {
assert.equal(json.v, 2);
assert.equal(json.type, 'chain');
assert.equal(json.network, network.type);
json.entries.forEach(function(entry) {
this._saveEntry(bcoin.chainblock.fromJSON(this, entry));
}, this);
};
/**
* Expose
*/

View File

@ -28,6 +28,8 @@ function ChainDB(chain, options) {
if (!options)
options = {};
EventEmitter.call(this);
this.options = options;
this.chain = chain;
this.file = options.file;
@ -35,12 +37,13 @@ function ChainDB(chain, options) {
if (!this.file)
this.file = process.env.HOME + '/bcoin-chain-' + network.type + '.db';
this._queue = [];
this.heightLookup = {};
this._queue = {};
this._cache = {};
this._bufferPool = { used: {} };
this._nullBlock = new Buffer(BLOCK_SIZE);
this._nullBlock.fill(0);
this.tip = -1;
this.highest = -1;
this.tip = null;
this.height = -1;
this.size = 0;
this.fd = null;
@ -55,6 +58,8 @@ function ChainDB(chain, options) {
this._init();
}
inherits(ChainDB, EventEmitter);
ChainDB.prototype._init = function _init() {
if (!bcoin.fs) {
utils.debug('`fs` module not available. Falling back to ramdisk.');
@ -85,6 +90,48 @@ ChainDB.prototype._init = function _init() {
this.fd = fs.openSync(this.file, 'r+');
};
ChainDB.prototype.load = function load(start, callback) {
var self = this;
var count = this.count();
var i = start || 0;
var lastEntry;
utils.debug('Starting chain load at height: %s', i);
function done(height) {
if (height != null) {
utils.debug(
'Blockchain is corrupt after height %d. Resetting.',
height);
self.resetHeight(height);
} else {
utils.debug('Chain successfully loaded.');
}
callback();
}
(function next() {
if (i >= count)
return done();
self.getAsync(i, function(err, entry) {
if (err)
return callback(err);
// Do some paranoid checks.
if (lastEntry && entry.prevBlock !== lastEntry.hash)
return done(Math.max(0, i - 2));
if (i % 10000 === 0)
utils.debug('Loaded %d blocks.', i);
lastEntry = entry;
i += 1;
next();
});
})();
};
ChainDB.prototype.closeSync = function closeSync() {
if (!bcoin.fs) {
this.ramdisk = null;
@ -167,14 +214,28 @@ ChainDB.prototype.count = function count() {
};
ChainDB.prototype.cache = function cache(entry) {
if (entry.height > this.tip) {
this.tip = entry.height;
if (entry.height > this.highest) {
this.highest = entry.height;
delete this._cache[entry.height - this._cacheWindow];
this._cache[entry.height] = entry;
assert(Object.keys(this._cache).length <= this._cacheWindow);
}
};
ChainDB.prototype.getHeight = function getHeight(hash) {
return this.heightLookup[hash];
};
ChainDB.prototype._populate = function _populate(entry) {
this.heightLookup[entry.hash] = entry.height;
if (!this.tip || entry.height > this.tip.height) {
this.tip = entry;
this.height = this.tip.height;
this.emit('tip', this.tip);
}
};
ChainDB.prototype.get = function get(height) {
return this.getSync(height);
};
@ -182,6 +243,9 @@ ChainDB.prototype.get = function get(height) {
ChainDB.prototype.getSync = function getSync(height) {
var data, entry;
if (typeof height === 'string')
height = this.heightLookup[height];
if (this._cache[height])
return this._cache[height];
@ -199,12 +263,10 @@ ChainDB.prototype.getSync = function getSync(height) {
if (!data)
return;
// Ignore if it is a null block.
if (utils.read32(data, 0) === 0)
return;
entry = bcoin.chainblock.fromRaw(this.chain, height, data);
this._populate(entry);
// Cache the past 1001 blocks in memory
// (necessary for isSuperMajority)
this.cache(entry);
@ -217,6 +279,9 @@ ChainDB.prototype.getAsync = function getAsync(height, callback) {
callback = utils.asyncify(callback);
if (typeof height === 'string')
height = this.heightLookup[height];
if (this._cache[height])
return callback(null, this._cache[height]);
@ -241,12 +306,10 @@ ChainDB.prototype.getAsync = function getAsync(height, callback) {
if (!data)
return callback();
// Ignore if it is a null block.
if (utils.read32(data, 0) === 0)
return callback();
entry = bcoin.chainblock.fromRaw(self.chain, height, data);
self._populate(entry);
// Cache the past 1001 blocks in memory
// (necessary for isSuperMajority)
self.cache(entry);
@ -260,13 +323,14 @@ ChainDB.prototype.save = function save(entry, callback) {
};
ChainDB.prototype.saveSync = function saveSync(entry) {
var self = this;
var raw, offset;
// Cache the past 1001 blocks in memory
// (necessary for isSuperMajority)
this.cache(entry);
this._populate(entry);
raw = entry.toRaw();
offset = entry.height * BLOCK_SIZE;
@ -283,6 +347,8 @@ ChainDB.prototype.saveAsync = function saveAsync(entry, callback) {
// (necessary for isSuperMajority)
this.cache(entry);
this._populate(entry);
// Something is already writing. Cancel it
// and synchronously write the data after
// it cancels.
@ -336,119 +402,131 @@ ChainDB.prototype.drop = function drop(height) {
delete this._cache[height];
};
ChainDB.prototype.remove = function remove(height) {
ChainDB.prototype.resetHeight = function resetHeight(height) {
if (typeof height === 'string')
height = this.heightLookup[height];
assert(height >= 0);
// Drop the queue and cache
this.drop(height);
// Write a null block
this._writeSync(this._nullBlock, height * BLOCK_SIZE);
// If we deleted several blocks at the end, go back
// to the last non-null block and truncate the file
// beyond that point.
if ((height + 1) * BLOCK_SIZE === this.size) {
while (this.isNull(height))
height--;
assert(height >= 0);
this.truncate(height);
}
return true;
};
ChainDB.prototype.truncate = function truncate(height) {
var size = (height + 1) * BLOCK_SIZE;
var count = this.count();
if (height === count - 1)
return;
assert(height <= count - 1);
assert(this.tip);
for (i = height + 1; i < count; i++) {
existing = this.get(i);
assert(existing);
this.drop(i);
delete this.heightLookup[existing.hash];
}
if (!bcoin.fs) {
this.ramdisk.truncate(size);
this.size = size;
this.tip = height;
this.highest = height;
this.tip = this.get(height);
assert(this.tip);
this.height = this.tip.height;
this.emit('tip', this.tip);
return;
}
fs.ftruncateSync(this.fd, size);
this.size = size;
this.tip = height;
this.highest = height;
this.tip = this.get(height);
assert(this.tip);
this.height = this.tip.height;
this.emit('tip', this.tip);
};
ChainDB.prototype.truncateAsync = function truncateAsync(height) {
ChainDB.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
var self = this;
var size = (height + 1) * BLOCK_SIZE;
var called;
if (typeof height === 'string')
height = this.heightLookup[height];
assert(height >= 0);
callback = utils.asyncify(callback);
if (!bcoin.fs) {
this.ramdisk.truncate(size);
this.size = size;
this.tip = height;
var size = (height + 1) * BLOCK_SIZE;
var count = this.count() - 1;
if (height === count - 1)
return callback();
assert(height <= count - 1);
assert(this.tip);
var pending = count - (height + 1);
for (i = height + 1; i < count; i++)
dropEntry(i);
function dropEntry(i) {
self.getAsync(i, function(err, existing) {
if (err)
return done(err);
assert(existing);
self.drop(i);
delete self.heightLookup[existing.hash];
if (!--pending)
done();
});
}
fs.ftruncate(this.fd, size, function(err) {
function done(err) {
if (called)
return;
called = true;
if (err)
return callback(err);
self.size = size;
self.tip = height;
if (!bcoin.fs) {
self.ramdisk.truncate(size);
self.size = size;
self.highest = height;
self.tip = self.get(height);
assert(self.tip);
self.height = self.tip.height;
self.emit('tip', self.tip);
return callback();
}
return callback();
});
};
fs.ftruncate(self.fd, size, function(err) {
if (err)
return callback(err);
ChainDB.prototype.isNull = function isNull(height) {
var data = this._readSync(4, height * BLOCK_SIZE);
if (!data)
return false;
return utils.read32(data, 0) === 0;
self.size = size;
self.highest = height;
self.tip = self.get(height);
assert(self.tip);
self.height = self.tip.height;
self.emit('tip', self.tip);
return callback();
});
}
};
ChainDB.prototype.has = function has(height) {
var data;
if (typeof height === 'string')
height = this.heightLookup[height];
if (this._queue[height] || this._cache[height])
if (height < 0 || height == null)
return false;
if ((height + 1) * BLOCK_SIZE <= this.size)
return true;
if (height < 0 || height == null)
return false;
if ((height + 1) * BLOCK_SIZE > this.size)
return false;
data = this._readSync(4, height * BLOCK_SIZE);
if (!data)
return false;
return utils.read32(data, 0) !== 0;
};
ChainDB.prototype.hasAsync = function hasAsync(height, callback) {
var data;
callback = utils.asyncify(callback);
if (this._queue[height] || this._cache[height])
return callback(null, true);
if (height < 0 || height == null)
return callback(null, false);
if ((height + 1) * BLOCK_SIZE > this.size)
return callback(null, false);
this._readAsync(4, height * BLOCK_SIZE, function(err, data) {
if (err)
return callback(err);
if (!data)
return callback(null, false);
return callback(null, utils.read32(data, 0) !== 0);
});
return false;
};
ChainDB.prototype._readSync = function _readSync(size, offset) {