blockchain db.

This commit is contained in:
Christopher Jeffrey 2016-01-20 17:40:51 -08:00
parent e461e44303
commit c111d673e7
2 changed files with 286 additions and 226 deletions

View File

@ -101,7 +101,7 @@ Block.prototype.abbr = function abbr() {
return this._raw.slice();
var res = new Array(80);
utils.writeU32(res, this.version, 0);
utils.write32(res, this.version, 0);
utils.copy(utils.toArray(this.prevBlock, 'hex'), res, 4);
utils.copy(utils.toArray(this.merkleRoot, 'hex'), res, 36);
utils.writeU32(res, this.ts, 68);

View File

@ -9,6 +9,7 @@ var EventEmitter = require('events').EventEmitter;
var bcoin = require('../bcoin');
var bn = require('bn.js');
var fs = require('fs');
var constants = bcoin.protocol.constants;
var network = bcoin.protocol.network;
var utils = bcoin.utils;
@ -28,6 +29,7 @@ function Chain(options) {
this.prefix = 'bt/chain/';
this.storage = this.options.storage;
this.strict = this.options.strict || false;
this.db = new ChainDB(this);
if (this.options.debug)
bcoin.debug = this.options.debug;
@ -42,12 +44,8 @@ function Chain(options) {
};
this.index = {
entries: [],
// Get hash by height
hashes: [],
// Get height by hash
heights: {},
count: 0,
count: this.db.count(),
lastTs: 0
};
@ -61,9 +59,11 @@ function Chain(options) {
{
hash: network.genesis.hash,
version: network.genesis.version,
// prevBlock: network.genesis.prevBlock,
prevBlock: network.genesis.prevBlock,
merkleRoot: network.genesis.merkleRoot,
ts: network.genesis.ts,
bits: network.genesis.bits,
nonce: network.genesis.nonce,
height: 0
}
]
@ -76,17 +76,12 @@ function Chain(options) {
else if (network.type === 'testnet')
this.fromJSON(require('./protocol/preload-test-full'));
this.resetHeight(+process.env.BCOIN_START_HEIGHT);
} else {
if (+process.env.BCOIN_NO_COMPACT !== 1) {
if (!this.options.fullNode)
this.fromJSON(network.preload);
}
}
this.tip = this.index.entries[this.index.entries.length - 1];
this.tip = this.db.get(this.index.count - 1);
// Last TS after preload, needed for fill percent
this.index.lastTs = this.index.entries[this.index.entries.length - 1].ts;
this.index.lastTs = this.tip.ts;
Chain.global = this;
@ -159,6 +154,9 @@ Chain.prototype._init = function _init() {
});
};
Chain.prototype.getEntry = function getEntry(height) {
};
Chain.prototype._addIndex = function _addIndex(entry, save) {
var self = this;
@ -169,7 +167,8 @@ Chain.prototype._addIndex = function _addIndex(entry, save) {
}
// Duplicate height
if (this.index.hashes[entry.height] === entry.hash)
var existing = this.db.get(entry.height);
if (existing && existing.hash === entry.hash)
return Chain.codes.unchanged;
// Fork at checkpoint
@ -186,16 +185,14 @@ Chain.prototype._addIndex = function _addIndex(entry, save) {
}
}
this.index.entries[entry.height] = entry;
this.index.hashes[entry.height] = entry.hash;
this.db.save(entry);
this.index.heights[entry.hash] = entry.height;
this.index.count++;
this.tip = this.index.entries[this.index.entries.length - 1];
this.emit('tip', this.tip);
if (!this.tip || entry.height > this.tip.height)
this.tip = entry;
if (save)
this._save(entry.hash, entry);
this.emit('tip', this.tip);
return Chain.codes.okay;
};
@ -219,13 +216,8 @@ Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) {
Chain.prototype.resetHeight = function resetHeight(height) {
var self = this;
var ahead = this.index.entries.slice(height + 1);
assert(height < this.index.entries.length);
// Nothing to do.
if (height === this.index.entries.length - 1)
return;
assert(height < this.index.count);
// Reset the orphan map completely. There may
// have been some orphans on a forked chain we
@ -235,27 +227,14 @@ Chain.prototype.resetHeight = function resetHeight(height) {
this.orphan.count = 0;
this.orphan.size = 0;
// Rebuild the index from our new (lower) height.
this.index.entries.length = height + 1;
for (var i = height + 1; height < this.index.count; i++) {
var existing = this.db.get(i);
this.db.del(i);
delete this.index.heights[existing.hash];
}
this.index.heights = this.index.entries.reduce(function(out, entry) {
if (!self.options.fullNode) {
if (!entry)
return out;
}
out[entry.hash] = entry.height;
return out;
}, {});
this.index.hashes.length = height + 1;
if (!this.options.fullNode)
this.index.count -= this._count(ahead);
else
this.index.count = height + 1;
// Set and emit our new (old) tip.
this.tip = this.index.entries[this.index.entries.length - 1];
this.tip = this.db.get(height);
this.index.count = height + 1;
this.emit('tip', this.tip);
// The lastTs is supposed to be the last ts
@ -264,17 +243,8 @@ Chain.prototype.resetHeight = function resetHeight(height) {
// be higher. Reset it if necessary.
this.index.lastTs = Math.min(
this.index.lastTs,
this.index.entries[this.index.entries.length - 1].ts
this.tip.ts
);
// Delete all the blocks now above us.
ahead.forEach(function(entry) {
if (!self.options.fullNode) {
if (!entry)
return;
}
self._delete(entry.hash);
});
};
Chain.prototype.resetTime = function resetTime(ts) {
@ -349,15 +319,18 @@ Chain.prototype.add = function add(block, peer) {
entry = new ChainBlock(this, {
hash: hash,
version: block.version,
// prevBlock: prevHash,
prevBlock: prevHash,
merkleRoot: block.merkleRoot,
ts: block.ts,
bits: block.bits,
nonce: block.nonce,
height: prevHeight + 1
});
// Add entry if we do not have it (or if
// there is another entry at its height)
if (this.index.hashes[entry.height] !== hash) {
var existing = this.db.get(entry.height);
if (!existing || existing.hash !== hash) {
assert(this.index.heights[entry.hash] == null);
// A valid block with an already existing
@ -365,17 +338,18 @@ Chain.prototype.add = function add(block, peer) {
// don't store by hash so we can't compare
// chainworks. We reset the chain, find a
// new peer, and wait to see who wins.
if (this.index.hashes[entry.height]) {
if (existing) {
// The tip has more chainwork, it is a
// higher height than the entry. This is
// not an alternate tip. Ignore it.
if (0)
if (this.tip.chainwork.cmp(entry.chainwork) > 0) {
code = Chain.codes.unchanged;
break;
}
// Get _our_ tip as opposed to
// the attempted alternate tip.
tip = this.index.entries[entry.height];
tip = existing;
// The block has equal chainwork (an
// alternate tip). Reset the chain, find
// a new peer, and wait to see who wins.
@ -466,13 +440,6 @@ Chain.prototype.add = function add(block, peer) {
this.orphan.size = 0;
}
// Potentially compact the chain here. A
// full chain is not necessary for spv.
// if (!this.options.fullNode) {
// if (this.size() > 100000)
// this.compact();
// }
if (code !== Chain.codes.okay) {
if (!(this.options.multiplePeers && code === Chain.codes.newOrphan))
utils.debug('Chain Error: %s', Chain.msg(code));
@ -492,7 +459,9 @@ Chain.prototype.has = function has(hash) {
};
Chain.prototype.byHeight = function byHeight(height) {
return this.index.entries[height] || null;
if (height == null)
return;
return this.db.get(height);
};
Chain.prototype.byHash = function byHash(hash) {
@ -505,13 +474,10 @@ Chain.prototype.byHash = function byHash(hash) {
};
Chain.prototype.byTime = function byTime(ts) {
for (var i = this.index.entries.length - 1; i >= 0; i--) {
if (!this.options.fullNode) {
if (!this.index.entries[i])
continue;
}
if (ts >= this.index.entries[i].ts)
return this.index.entries[i];
for (var i = this.index.count - 1; i >= 0; i--) {
var existing = this.db.get(i);
if (ts >= existing.ts)
return existing;
}
return null;
};
@ -540,11 +506,11 @@ Chain.prototype.getOrphan = function getOrphan(hash) {
};
Chain.prototype.getTip = function getTip() {
return this.index.entries[this.index.entries.length - 1];
return this.tip;
};
Chain.prototype.isFull = function isFull() {
var last = this.index.entries[this.index.entries.length - 1].ts;
var last = this.tip.ts;
var delta = utils.now() - last;
return delta < 40 * 60;
};
@ -564,18 +530,15 @@ Chain.prototype.hashRange = function hashRange(start, end) {
if (!start || !end)
return [];
hashes = this.index.hashes.slice(start.height, end.height + 1);
if (!this.options.fullNode)
hashes = this._filter(hashes);
for (var i = start.height; i < end.height + 1; i++)
hashes.push(this.db.get(i).hash);
return hashes;
};
Chain.prototype.locatorHashes = function locatorHashes(start) {
var chain = this.index.hashes;
var hashes = [];
var top = chain.length - 1;
var top = this.height();
var step = 1;
var i;
@ -595,22 +558,23 @@ Chain.prototype.locatorHashes = function locatorHashes(start) {
// is our tip. This is useful for getheaders
// when not using headers-first.
hashes.push(start);
top = chain.length - 1;
top = this.index.count - 1;
}
} else if (typeof start === 'number') {
top = start;
}
assert(chain[top]);
// assert(chain[top]);
i = top;
for (;;) {
if (chain[i])
hashes.push(chain[i]);
var existing = this.db.get(i);
if (existing)
hashes.push(existing.hash);
i = i - step;
if (i <= 0) {
if (i + step !== 0)
hashes.push(chain[0]);
hashes.push(this.db.get(0).hash);
break;
}
if (hashes.length >= 10)
@ -694,12 +658,7 @@ Chain.prototype.target = function target(last, block) {
}
// Back 2 weeks
// first = last;
// for (i = 0; first && i < network.powDiffInterval - 1; i++)
// first = first.prev;
// Back 2 weeks
first = this.index.entries[last.height - (network.powDiffInterval - 1)];
first = this.db.get(last.height - (network.powDiffInterval - 1));
assert(first);
@ -731,117 +690,9 @@ Chain.prototype.retarget = function retarget(last, first) {
return utils.toCompact(target);
};
Chain.prototype.compact = function compact(keep) {
var entries;
if (+process.env.BCOIN_NO_COMPACT === 1)
return;
entries = this._compact(keep);
this.index.entries = {};
this.index.hashes = [];
this.index.heights = {};
this.index.count = 0;
json.entries.forEach(function(entry) {
this._addIndex(ChainBlock.fromJSON(this, entry));
}, this);
};
Chain.prototype._compact = function _compact(keep) {
var entries;
var last, first, delta1, delta2, delta3, lastTs, lastHeight;
var i, ts, delta, hdelta;
if (+process.env.BCOIN_NO_COMPACT === 1)
return this.index.entries;
entries = this._filter(this.index.entries);
if (!keep)
keep = network.powDiffInterval + 10;
// Keep only last ~2016 consequent blocks, dilate others at:
// 7 day range for blocks before 2013
// 12 hour for blocks before 2014
// 6 hour for blocks in 2014 and after it
// (or at maximum 250 block range)
last = entries.slice(-keep);
first = [];
delta1 = 7 * 24 * 3600;
delta2 = 12 * 3600;
delta3 = 6 * 3600;
lastTs = 0;
lastHeight = -1000;
for (i = 0; i < entries.length - keep; i++) {
ts = entries[i].ts;
delta = ts < 1356984000
? delta1
: ts < 1388520000 ? delta2 : delta3;
hdelta = entries[i].height - lastHeight;
if (ts - lastTs < delta && hdelta < 250)
continue;
lastTs = ts;
lastHeight = entries[i].height;
first.push(this.index.entries[i]);
}
return first.concat(last);
};
Chain.prototype._save = function(hash, obj) {
var self = this;
if (!this.storage)
return;
this.storage.put(this.prefix + hash, obj.toJSON(), function(err) {
if (err)
self.emit('error', err);
});
};
Chain.prototype._delete = function(hash) {
var self = this;
if (!this.storage)
return;
this.storage.del(this.prefix + hash, function(err) {
if (err)
self.emit('error', err);
});
};
Chain.prototype._count = function(obj) {
for (var i = 0, c = 0; i < obj.length; i++)
if (obj[i])
c++;
return c;
};
Chain.prototype._filter = function(obj) {
for (var i = 0, a = []; i < obj.length; i++)
if (obj[i])
a.push(obj[i]);
return a;
};
Chain.prototype.toJSON = function toJSON() {
var entries = this.index.entries;
if (!this.options.fullNode)
entries = this._compact();
return {
v: 2,
type: 'chain',
@ -858,10 +709,200 @@ Chain.prototype.fromJSON = function fromJSON(json) {
assert.equal(json.network, network.type);
json.entries.forEach(function(entry) {
this._addIndex(ChainBlock.fromJSON(this, entry));
this._addIndex(ChainBlock.fromJSON(this, entry), true);
}, this);
};
assert(this.index.entries.length > 0);
/**
* ChainDB
*/
var BLOCK_SIZE = 80;
// var BLOCK_SIZE = 144;
function ChainDB(chain) {
this.file = process.env.HOME + '/bcoin-' + network.type + '.blockchain';
try {
fs.unlinkSync(this.file);
} catch (e) {
}
var exists = false;
try {
fs.accessSync(file);
exists = true;
} catch (e) {
exists = false;
}
if (!exists)
fs.writeFileSync(this.file, new Buffer(0));
this.fd = fs.openSync(this.file, 'r+');
this.chain = chain;
this._buffer = [];
this._cache = {};
this._bufferPool = {};
this._nullBlock = new Buffer(BLOCK_SIZE);
this._nullBlock.fill(0);
}
ChainDB.prototype.getBuffer = function(size) {
if (!this._bufferPool[size])
this._bufferPool[size] = new Buffer(size);
return this._bufferPool[size];
};
ChainDB.prototype.save = function save(entry) {
// Cache the past 2016 blocks in memory
this._cache[entry.height] = entry;
delete this._cache[entry.height - network.powDiffInterval];
assert(Object.keys(this._cache).length < network.powDiffInterval + 1);
return this._write(new Buffer(entry.toRaw()), entry.height * BLOCK_SIZE);
};
ChainDB.prototype._write = function _write(data, offset) {
var size, bytes, index, hash;
var self = this;
if (offset < 0 || offset == null)
return false;
hash = offset;
size = data.length;
bytes = 0;
index = 0;
// Something is already writing. Cancel it
// and synchronously write the data after
// it cancels.
if (this._buffer[hash]) {
assert(this._buffer[hash].data.length === data.length);
this._buffer[hash].data = data;
this._buffer[hash].queue.push([data, offset]);
return;
}
// Speed up writes by doing them asynchronously
// and keeping the data to be written in memory.
this._buffer[hash] = { queue: [], data: data };
(function callee() {
fs.write(self.fd, data, index, size, offset, function(err, bytes) {
// Something tried to write here but couldn't.
// Synchronously write it and get it over with.
if (self._buffer[hash].queue.length) {
self._buffer[hash].queue.forEach(function(chunk) {
self._writeSync(chunk[0], chunk[1]);
});
delete self._buffer[hash];
return;
}
index += bytes;
size -= bytes;
offset += bytes;
if (index === data.length) {
delete self._buffer[hash];
return;
}
callee();
});
})();
return false;
};
ChainDB.prototype._writeSync = function _writeSync(data, offset) {
var size, bytes, index;
if (offset < 0 || offset == null)
return false;
size = data.length;
bytes = 0;
index = 0;
while (bytes = fs.writeSync(this.fd, data, index, size, offset)) {
index += bytes;
size -= bytes;
offset += bytes;
if (index === data.length)
return true;
}
return false;
};
ChainDB.prototype.del = function del(height) {
this._write(this._nullBlock, height * BLOCK_SIZE);
if (height * BLOCK_SIZE + BLOCK_SIZE === this.size()) {
while (this.isNull(height))
height--;
fs.ftruncateSync(this.fd, height * BLOCK_SIZE + BLOCK_SIZE);
}
return true;
};
ChainDB.prototype.isNull = function isNull(height) {
var data = this._read(1, height * BLOCK_SIZE);
if (!data)
return false;
return utils.read32(data, 0) === 0;
};
ChainDB.prototype._read = function _read(size, offset) {
var data = this.getBuffer(size);
var bytes = 0;
var index = 0;
if (offset < 0 || offset == null)
return;
if (this._buffer[offset]) {
assert(this._buffer[offset].data.length >= size);
return this._buffer[offset].data.slice(0, size);
}
try {
while (bytes = fs.readSync(this.fd, data, index, size, offset)) {
index += bytes;
size -= bytes;
offset += bytes;
if (index === data.length) {
if (utils.read32(data, 0) === 0)
return;
return data;
}
}
} catch (e) {
return;
}
};
ChainDB.prototype.size = function size() {
try {
return fs.statSync(this.file).size;
} catch (e) {
return 0;
}
};
ChainDB.prototype.count = function count() {
return this.size() / BLOCK_SIZE | 0;
};
ChainDB.prototype.get = function get(height) {
var data;
if (this._cache[height])
return this._cache[height];
data = this._read(BLOCK_SIZE, height * BLOCK_SIZE);
if (!data)
return;
return ChainBlock.fromRaw(this.chain, height, data);
};
/**
@ -872,19 +913,22 @@ function ChainBlock(chain, data) {
this.chain = chain;
this.hash = data.hash;
this.version = data.version;
// this.prevBlock = data.prevBlock;
this.prevBlock = data.prevBlock;
this.merkleRoot = data.merkleRoot;
this.ts = data.ts;
this.bits = data.bits;
this.nonce = data.nonce;
this.height = data.height;
this.chainwork = this.getChainwork();
this.chainwork = data.chainwork || new bn(0);
// this.chainwork = data.chainwork || this.getChainwork();
}
ChainBlock.prototype.__defineGetter__('prev', function() {
return this.chain.index.entries[this.height - 1];
return this.chain.db.get(this.height - 1);
});
ChainBlock.prototype.__defineGetter__('next', function() {
return this.chain.index.entries[this.height + 1];
return this.chain.db.get(this.height + 1);
});
ChainBlock.prototype.__defineGetter__('proof', function() {
@ -939,36 +983,52 @@ ChainBlock.prototype.isSuperMajority = function(version, required) {
};
ChainBlock.prototype.toJSON = function() {
// return [
// this.hash,
// this.version,
// this.prevBlock,
// this.ts,
// this.bits,
// this.height
// };
return {
hash: this.hash,
version: this.version,
// prevBlock: this.prevBlock,
prevBlock: this.prevBlock,
merkleRoot: this.merkleRoot,
ts: this.ts,
bits: this.bits,
nonce: this.nonce,
height: this.height
};
};
ChainBlock.fromJSON = function(chain, json) {
// return new ChainBlock(chain, {
// hash: json[0],
// version: json[1],
// prevBlock: json[2],
// ts: json[3],
// bits: json[4],
// height: json[5]
// });
return new ChainBlock(chain, json);
};
ChainBlock.prototype.toRaw = function toRaw() {
var res = new Array(BLOCK_SIZE);
utils.writeU32(res, this.version, 0);
utils.copy(utils.toArray(this.prevBlock, 'hex'), res, 4);
utils.copy(utils.toArray(this.merkleRoot, 'hex'), res, 36);
utils.writeU32(res, this.ts, 68);
utils.writeU32(res, this.bits, 72);
utils.writeU32(res, this.nonce, 76);
// utils.copy(utils.toArray(this.hash, 'hex'), res, 80);
// utils.copy(this.chainwork.toArray('be', 32), res, 112);
return res;
};
ChainBlock.fromRaw = function fromRaw(chain, height, p) {
return new ChainBlock(chain, {
hash: utils.toHex(utils.dsha256(p.slice(0, 80))),
version: utils.read32(p, 0),
prevBlock: utils.toHex(utils.toArray(p.slice(4, 36))),
merkleRoot: utils.toHex(utils.toArray(p.slice(36, 68))),
ts: utils.readU32(p, 68),
bits: utils.readU32(p, 72),
nonce: utils.readU32(p, 76),
height: height,
// hash: utils.toHex(utils.toArray(p.slice(80, 112))),
// chainwork: new bn(p.slice(112, 144))
});
};
/**
* Expose
*/