RanchiMall/flocore-node
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Chris Kleeschulte 2017-07-19 18:58:18 -04:00
parent de4c59f958
commit 6b45ef27cd
5 changed files with 142 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
lib/services/block/encoding.js
View File

@ -5,18 +5,16 @@ var Block = require('bitcore-lib').Block;
function Encoding(servicePrefix) { function Encoding(servicePrefix) {
this._servicePrefix = servicePrefix; this._servicePrefix = servicePrefix;
this._blockPrefix = new Buffer('00', 'hex');
this._metaPrefix = new Buffer('01', 'hex');
} }
// ---- hash --> rawblock // ---- hash --> rawblock
Encoding.prototype.encodeBlockKey = function(hash) { Encoding.prototype.encodeBlockKey = function(hash) {
return Buffer.concat([ this._servicePrefix, this._blockPrefix, new Buffer(hash, 'hex') ]); return Buffer.concat([ this._servicePrefix, new Buffer(hash, 'hex') ]);
}; };
Encoding.prototype.decodeBlockKey = function(buffer) { Encoding.prototype.decodeBlockKey = function(buffer) {
return buffer.slice(3).toString('hex'); return buffer.slice(2).toString('hex');
}; };
Encoding.prototype.encodeBlockValue = function(block) { Encoding.prototype.encodeBlockValue = function(block) {
@ -27,27 +25,4 @@ Encoding.prototype.decodeBlockValue = function(buffer) {
return Block.fromBuffer(buffer); return Block.fromBuffer(buffer);
}; };
// ---- height --> hash, chainwork
Encoding.prototype.encodeMetaKey = function(height) {
var heightBuf = new Buffer(4);
heightBuf.writeUInt32BE(height);
return Buffer.concat([ this._servicePrefix, this._metaPrefix, heightBuf ]);
};
Encoding.prototype.decodeMetaKey = function(buffer) {
return buffer.readUInt32BE(3);
};
Encoding.prototype.encodeMetaValue = function(value) {
// { chainwork: hex-string, hash: hex-string }
return Buffer.concat([ new Buffer(value.hash, 'hex'), new Buffer(value.chainwork, 'hex') ]);
};
Encoding.prototype.decodeMetaValue = function(buffer) {
return {
hash: buffer.slice(0, 32).toString('hex'),
chainwork: buffer.slice(32).toString('hex')
};
};
module.exports = Encoding; module.exports = Encoding;

148
lib/services/block/index.js
View File

@ -11,7 +11,6 @@ var utils = require('../../utils');
var _ = require('lodash'); var _ = require('lodash');
var assert = require('assert'); var assert = require('assert');
var BN = require('bn.js'); var BN = require('bn.js');
var consensus = require('bcoin').consensus;
var constants = require('../../constants'); var constants = require('../../constants');
var BlockService = function(options) { var BlockService = function(options) {
@ -21,18 +20,12 @@ var BlockService = function(options) {
this._tip = null; this._tip = null;
this._p2p = this.node.services.p2p; this._p2p = this.node.services.p2p;
this._db = this.node.services.db; this._db = this.node.services.db;
this._header = this.node.services.header;
this._subscriptions = {}; this._subscriptions = {};
this._subscriptions.block = []; this._subscriptions.block = [];
this._subscriptions.reorg = []; this._subscriptions.reorg = [];
//memory
this._maxMem = options.maxMemory || 1500; // in MB
this._deferTimeout = null;
// meta is [{ chainwork: chainwork, hash: hash }]
this._meta = [];
// in-memory full/raw block cache // in-memory full/raw block cache
this._blockQueue = LRU({ this._blockQueue = LRU({
max: 50 * (1 * 1024 * 1024), // 50 MB of blocks, max: 50 * (1 * 1024 * 1024), // 50 MB of blocks,
@ -53,9 +46,8 @@ var BlockService = function(options) {
inherits(BlockService, BaseService); inherits(BlockService, BaseService);
BlockService.dependencies = [ 'p2p', 'db' ]; BlockService.dependencies = [ 'p2p', 'db', 'header' ];
BlockService.MAX_CHAINWORK = new BN(1).ushln(256);
BlockService.MAX_BLOCKS = 500; BlockService.MAX_BLOCKS = 500;
// --- public prototype functions // --- public prototype functions
@ -71,7 +63,8 @@ BlockService.prototype.getAPIMethods = function() {
}; };
BlockService.prototype.getBestBlockHash = function(callback) { BlockService.prototype.getBestBlockHash = function(callback) {
return callback(null, this._meta[this._meta.length - 1].hash); var headers = this._header.getAllHeaders();
return headers[headers.length - 1].hash;
}; };
BlockService.prototype.getBlock = function(arg, callback) { BlockService.prototype.getBlock = function(arg, callback) {
@ -95,6 +88,7 @@ BlockService.prototype.getBlockHeader = function(blockArg, callback) {
return callback(); return callback();
} }
var headers = this._header.getAllHeaders();
this._getBlock(blockArg, function(err, block) { this._getBlock(blockArg, function(err, block) {
if(err) { if(err) {
@ -188,15 +182,17 @@ BlockService.prototype.start = function(callback) {
function(tip, next) { function(tip, next) {
self._tip = tip; self._tip = tip;
self._chainTips.push(self._tip.hash); self._chainTips.push(self._tip.hash);
self._loadMeta(next);
} }
], function(err) { ], function(err) {
if(err) { if(err) {
return callback(err); return callback(err);
} }
self._setListeners(); self._setListeners();
self._startSubscriptions(); self._startSubscriptions();
callback(); callback();
}); });
}; };
@ -270,24 +266,6 @@ BlockService.prototype._cacheBlock = function(block) {
}; };
BlockService.prototype._computeChainwork = function(bits, prev) {
var target = consensus.fromCompact(bits);
if (target.isNeg() || target.cmpn(0) === 0) {
return new BN(0);
}
var proof = BlockService.MAX_CHAINWORK.div(target.iaddn(1));
if (!prev) {
return proof;
}
return proof.iadd(prev);
};
BlockService.prototype._determineBlockState = function(block) { BlockService.prototype._determineBlockState = function(block) {
if (this._isOutOfOrder(block)) { if (this._isOutOfOrder(block)) {
@ -365,16 +343,6 @@ BlockService.prototype._getBlockOperations = function(block) {
}; };
BlockService.prototype._getChainwork = function(tipHash) {
var block = this._blockQueue.get(tipHash);
var lastChainwork = this._meta[this._meta.length - 1].chainwork;
var prevChainwork = new BN(new Buffer(lastChainwork, 'hex'));
return this._computeChainwork(block.header.bits, prevChainwork);
};
BlockService.prototype._getDelta = function(tip) { BlockService.prototype._getDelta = function(tip) {
var blocks = []; var blocks = [];
@ -393,8 +361,9 @@ BlockService.prototype._getDelta = function(tip) {
BlockService.prototype._getHash = function(blockArg) { BlockService.prototype._getHash = function(blockArg) {
var headers = this._header.getAllHeaders();
return (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) && return (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) &&
this._meta[blockArg] ? this._meta[blockArg] : null; headers[blockArg] ? headers[blockArg] : null;
}; };
@ -411,15 +380,15 @@ BlockService.prototype._getIncompleteChainIndexes = function(block) {
}; };
BlockService.prototype._getOldBlocks = function(currentHash, commonAncestorHash) { BlockService.prototype._getOldBlocks = function(currentHash, commonAncestorHash) {
// the old blocks should be in the meta colection
if (currentHash === commonAncestorHash || !commonAncestorHash || !currentHash) { if (currentHash === commonAncestorHash || !commonAncestorHash || !currentHash) {
return; return;
} }
var oldBlocks; var oldBlocks;
for(var i = this._meta.length - 1; i > 0; --i) { var headers = this._header.getAllHeaders();
var item = this._meta[i]; for(var i = headers.length - 1; i > 0; --i) {
var item = headers[i];
if (item.hash === currentHash) { if (item.hash === currentHash) {
oldBlocks = [this._blockQueue.get(currentHash)]; oldBlocks = [this._blockQueue.get(currentHash)];
continue; continue;
@ -472,28 +441,14 @@ BlockService.prototype._onReorg = function(oldBlockList, newBlockList, commonAnc
// remove all the old blocks that we reorg from // remove all the old blocks that we reorg from
oldBlockList.forEach(function(block) { oldBlockList.forEach(function(block) {
removalOps.concat([ removalOps.push({
{ type: 'del',
type: 'del', key: this.encoding.encodeBlockKey(block.header.timestamp),
key: this.encoding.encodeBlockKey(block.header.timestamp), });
},
{
type: 'del',
key: this.encoding.encodeMetaKey(block.header.height),
}
]);
}); });
this._db.batch(removalOps); this._db.batch(removalOps);
// remove the blocks from the in-memory meta list
var newMetaHeight = commonAncestor.header.height;
for(var i = this._meta.length - 1; i > 0; --i) {
if (i > newMetaHeight) {
this._meta.splice(i, 1);
}
}
//call onBlock for each of the new blocks //call onBlock for each of the new blocks
newBlockList.forEach(this._onBlock.bind(this)); newBlockList.forEach(this._onBlock.bind(this));
// if the common ancestor block height is greater than our own, then nothing to do for the reorg // if the common ancestor block height is greater than our own, then nothing to do for the reorg
@ -528,32 +483,8 @@ BlockService.prototype._isOutOfOrder = function(block) {
}; };
BlockService.prototype._loadMeta = function(callback) { BlockService.prototype._onAllHeaders = function(headers) {
var self = this; this._bestHeight = headers.length;
var criteria = {
gte: self._encoding.encodeMetaKey(0),
lte: self._encoding.encodeMetaKey(0xffffffff)
};
var stream = this._db.createReadStream(criteria);
stream.on('end', function() {
if (self._meta.length < 1) {
self._meta.push({
chainwork: '0000000000000000000000000000000000000000000000000000000100010001',
hash: self.GENESIS_HASH
});
}
callback();
});
stream.on('data', function(data) {
self._meta.push(self._encoding.decodeMetaValue(data.value));
});
};
BlockService.prototype._onBestHeight = function(height) {
this._bestHeight = height;
this._startSync(); this._startSync();
}; };
@ -591,45 +522,11 @@ BlockService.prototype._onBlock = function(block) {
break; break;
default: default:
// send all unsent blocks now that we have a complete chain // send all unsent blocks now that we have a complete chain
this._saveMetaData(block);
this._sendDelta(); this._sendDelta();
break; break;
} }
}; };
BlockService.prototype._saveMetaData = function(block) {
var item = {
chainwork: this._getChainwork(block.hash).toString(16, 64),
hash: block.hash
};
this._meta.push(item);
var operations = [];
// tip
this._tip.hash = block.hash;
this._tip.height = this._meta.length;
var tipInfo = utils.encodeTip(this._tip, this.name);
operations.push({
type: 'put',
key: tipInfo.key,
value: tipInfo.value
});
//meta positions in db
operations.push({
type: 'put',
key: this._encoding.encodeMetaKey(this._meta.length - 1),
value: this._encoding.encodeMetaValue(item)
});
this._db.batch(operations);
};
BlockService.prototype._selectActiveChain = function() { BlockService.prototype._selectActiveChain = function() {
var chainTip; var chainTip;
@ -664,8 +561,9 @@ BlockService.prototype._sendDelta = function() {
this._broadcast(this._subscriptions.block, 'block/block', blocks[i]); this._broadcast(this._subscriptions.block, 'block/block', blocks[i]);
} }
var len = this._meta.length - 1; var headers = this._header.getAllHeaders();
this._setTip({ height: len, hash: this._meta[len].hash }); var len = headers.length - 1;
this._setTip({ height: len, hash: headers[len].hash });
if (++this._blockCount >= BlockService.MAX_BLOCKS) { if (++this._blockCount >= BlockService.MAX_BLOCKS) {
this._latestBlockHash = this._tip.hash; this._latestBlockHash = this._tip.hash;
@ -710,7 +608,7 @@ BlockService.prototype._continueSync = function() {
BlockService.prototype._setListeners = function() { BlockService.prototype._setListeners = function() {
this._p2p.once('bestHeight', this._onBestHeight.bind(this)); this._header.once('headers', this._onAllHeaders.bind(this));
}; };

21
lib/services/header/encoding.js
View File

@ -7,12 +7,20 @@ function Encoding(servicePrefix) {
// ---- hash --> header // ---- hash --> header
Encoding.prototype.encodeHeaderKey = function(hash) { Encoding.prototype.encodeHeaderKey = function(height, hash) {
return Buffer.concat([ this._servicePrefix, new Buffer(hash, 'hex') ]); var heightBuf = new Buffer(4);
heightBuf.writeUInt32BE(height);
var hashBuf = new Buffer(hash || new Array(65).join('0'), 'hex');
return Buffer.concat([ this._servicePrefix, heightBuf, hashBuf ]);
}; };
Encoding.prototype.decodeHeaderKey = function(buffer) { Encoding.prototype.decodeHeaderKey = function(buffer) {
return buffer.slice(2).toString('hex'); var height = buffer.readUInt32BE(2);
return {
height: height,
hash: buffer.slice(6).toString('hex')
};
}; };
Encoding.prototype.encodeHeaderValue = function(header) { Encoding.prototype.encodeHeaderValue = function(header) {
@ -28,7 +36,8 @@ Encoding.prototype.encodeHeaderValue = function(header) {
nonceBuf.writeUInt32BE(header.nonce); nonceBuf.writeUInt32BE(header.nonce);
var heightBuf = new Buffer(4); var heightBuf = new Buffer(4);
heightBuf.writeUInt32BE(header.height); heightBuf.writeUInt32BE(header.height);
return Buffer.concat([ versionBuf, prevHash, merkleRoot, tsBuf, bitsBuf, nonceBuf, heightBuf ]); var chainworkBuf = new Buffer(header.chainwork, 'hex');
return Buffer.concat([ versionBuf, prevHash.reverse(), merkleRoot.reverse(), tsBuf, bitsBuf, nonceBuf, heightBuf, chainworkBuf ]);
}; };
Encoding.prototype.decodeHeaderValue = function(buffer) { Encoding.prototype.decodeHeaderValue = function(buffer) {
@ -39,6 +48,7 @@ Encoding.prototype.decodeHeaderValue = function(buffer) {
var bits = buffer.readUInt32BE(72); var bits = buffer.readUInt32BE(72);
var nonce = buffer.readUInt32BE(76); var nonce = buffer.readUInt32BE(76);
var height = buffer.readUInt32BE(80); var height = buffer.readUInt32BE(80);
var chainwork = buffer.slice(84).toString('hex');
return { return {
version: version, version: version,
prevHash: prevHash, prevHash: prevHash,
@ -46,7 +56,8 @@ Encoding.prototype.decodeHeaderValue = function(buffer) {
timestamp: ts, timestamp: ts,
bits: bits, bits: bits,
nonce: nonce, nonce: nonce,
height: height height: height,
chainwork: chainwork
}; };
}; };

100
lib/services/header/index.js
View File

@ -6,6 +6,9 @@ var Encoding = require('./encoding');
var index = require('../../'); var index = require('../../');
var log = index.log; var log = index.log;
var utils = require('../../utils'); var utils = require('../../utils');
var async = require('async');
var BN = require('bn.js');
var consensus = require('bcoin').consensus;
var HeaderService = function(options) { var HeaderService = function(options) {
@ -14,48 +17,61 @@ var HeaderService = function(options) {
this._tip = null; this._tip = null;
this._p2p = this.node.services.p2p; this._p2p = this.node.services.p2p;
this._db = this.node.services.db; this._db = this.node.services.db;
this._headers = [];
}; };
inherits(HeaderService, BaseService); inherits(HeaderService, BaseService);
HeaderService.dependencies = [ 'p2p', 'db' ]; HeaderService.dependencies = [ 'p2p', 'db' ];
HeaderService.MAX_CHAINWORK = new BN(1).ushln(256);
HeaderService.STARTING_CHAINWORK = '0000000000000000000000000000000000000000000000000000000100010001';
// --- public prototype functions // --- public prototype functions
HeaderService.prototype.getAPIMethods = function() { HeaderService.prototype.getAPIMethods = function() {
var methods = [ var methods = [
['getAllHeaders', this, this.getAllHeaders, 0] ['getAllHeaders', this, this.getAllHeaders, 0],
['getBestHeight', this, this.getBestHeight, 0]
]; ];
return methods; return methods;
}; };
HeaderService.prototype.getBestHeight = function() {
return this._tip.height;
};
HeaderService.prototype.start = function(callback) { HeaderService.prototype.start = function(callback) {
var self = this; var self = this;
self._db.getPrefix(self.name, function(err, prefix) { async.waterfall([
function(next) {
self._db.getPrefix(self.name, next);
},
function(prefix, next) {
self._encoding = new Encoding(prefix);
self._db.getServiceTip(self.name, next);
},
function(tip, next) {
self._tip = tip;
self._getPersistedHeaders(next);
}
], function(err, headers) {
if(err) { if (err) {
return callback(err); return callback(err);
} }
self._db.getServiceTip(self.name, function(err, tip) { this._headers = headers;
self._setListeners();
self._startSubscriptions();
callback();
if(err) {
return callback(err);
}
self._tip = tip;
self._encoding = new Encoding(prefix);
self._setListeners();
self._startSubscriptions();
callback();
});
}); });
}; };
HeaderService.prototype.stop = function(callback) { HeaderService.prototype.stop = function(callback) {
@ -74,7 +90,15 @@ HeaderService.prototype._startSubscriptions = function() {
} }
this._bus.on('p2p/headers', this._onHeaders.bind(this)); this._bus.on('p2p/headers', this._onHeaders.bind(this));
this._bus.on('p2p/block', this._onHeaders.bind(this));
this._bus.subscribe('p2p/headers'); this._bus.subscribe('p2p/headers');
this._bus.subscribe('p2p/block');
};
HeaderService.prototype._onBlock = function(block) {
// we just want the header to keep a running list
this._onHeaders([block.header]);
}; };
HeaderService.prototype._onHeaders = function(headers) { HeaderService.prototype._onHeaders = function(headers) {
@ -83,11 +107,11 @@ HeaderService.prototype._onHeaders = function(headers) {
return; return;
} }
var operations = this._getHeaderOperations(headers);
this._tip.hash = headers[headers.length - 1].hash; this._tip.hash = headers[headers.length - 1].hash;
this._tip.height = this._tip.height + headers.length; this._tip.height = this._tip.height + headers.length;
var operations = this._getHeaderOperations(headers);
var tipOps = utils.encodeTip(this._tip, this.name); var tipOps = utils.encodeTip(this._tip, this.name);
operations.push({ operations.push({
@ -97,10 +121,11 @@ HeaderService.prototype._onHeaders = function(headers) {
}); });
this._db.batch(operations); this._db.batch(operations);
this._headers.concat(headers);
if (this._tip.height >= this._bestHeight) { if (this._tip.height >= this._bestHeight) {
log.info('Header download complete.'); log.info('Header download complete.');
this.emit('headers'); this.emit('headers', this._headers);
return; return;
} }
@ -112,11 +137,14 @@ HeaderService.prototype._getHeaderOperations = function(headers) {
var self = this; var self = this;
var runningHeight = this._tip.height; var runningHeight = this._tip.height;
var prevHeader = this._headers[this._headers.length - 1];
return headers.map(function(header) { return headers.map(function(header) {
header.height = ++runningHeight; header.height = ++runningHeight;
header.chainwork = self._getChainwork(header, prevHeader).toString(16, 32);
prevHeader = header;
return { return {
type: 'put', type: 'put',
key: self._encoding.encodeHeaderKey(header.hash), key: self._encoding.encodeHeaderKey(header.height, header.hash),
value: self._encoding.encodeHeaderValue(header) value: self._encoding.encodeHeaderValue(header)
}; };
}); });
@ -163,7 +191,11 @@ HeaderService.prototype._sync = function() {
}; };
HeaderService.prototype.getAllHeaders = function(callback) { HeaderService.prototype.getAllHeaders = function() {
return this._headers;
};
HeaderService.prototype._getPersistedHeaders = function(callback) {
var self = this; var self = this;
var results = []; var results = [];
@ -183,7 +215,7 @@ HeaderService.prototype.getAllHeaders = function(callback) {
stream.on('data', function(data) { stream.on('data', function(data) {
var res = {}; var res = {};
res[self._encoding.decodeHeaderKey(data.key)] = self._encoding.decodeHeaderValue(data.value); res[self._encoding.decodeHeaderKey(data.key).hash] = self._encoding.decodeHeaderValue(data.value);
results.push(res); results.push(res);
}); });
@ -196,5 +228,31 @@ HeaderService.prototype.getAllHeaders = function(callback) {
}; };
HeaderService.prototype._getChainwork = function(header, prevHeader) {
var lastChainwork = prevHeader ? prevHeader.chainwork : HeaderService.STARTING_CHAINWORK;
var prevChainwork = new BN(new Buffer(lastChainwork, 'hex'));
return this._computeChainwork(header.bits, prevChainwork);
};
HeaderService.prototype._computeChainwork = function(bits, prev) {
var target = consensus.fromCompact(bits);
if (target.isNeg() || target.cmpn(0) === 0) {
return new BN(0);
}
var proof = HeaderService.MAX_CHAINWORK.div(target.iaddn(1));
if (!prev) {
return proof;
}
return proof.iadd(prev);
};
module.exports = HeaderService; module.exports = HeaderService;

35
lib/services/p2p/index.js
View File

@ -22,7 +22,7 @@ var P2P = function(options) {
this._initP2P(); this._initP2P();
this._initPubSub(); this._initPubSub();
this._bcoin = null; this._bcoin = null;
this._currentBestHeight = null;
}; };
util.inherits(P2P, BaseService); util.inherits(P2P, BaseService);
@ -41,6 +41,7 @@ P2P.prototype.getAPIMethods = function() {
['getInfo', this, this.getInfo, 0], ['getInfo', this, this.getInfo, 0],
['getMempool', this, this.getMempool, 0], ['getMempool', this, this.getMempool, 0],
['sendTransaction', this, this.sendTransaction, 1] ['sendTransaction', this, this.sendTransaction, 1]
['getBestHeight', this, this.getBestHeight, 0]
]; ];
return methods; return methods;
}; };
@ -171,23 +172,31 @@ P2P.prototype._connect = function() {
} }
}; };
P2P.prototype._getBestHeight = function(peer) { P2P.prototype._getBestHeight = function() {
if (this._peers < this._minPeers) {
this._peerHeights.push(peer.bestHeight); return 0;
if (this._peerHeights.length >= this._minPeers) {
return Math.max(...this._peerHeights);
} }
var maxHeight = 0;
for(var i = 0; i < this._peers.length; i++) {
if (this._peers[i].bestHeight > maxHeight) {
maxHeight = this._peers[i].bestHeight;
this._peer = this._peers[i];
}
}
return maxHeight;
}; };
// we should only choose from a list of peers that sync'ed
P2P.prototype._getPeer = function() { P2P.prototype._getPeer = function() {
if (this._peers.length === 0) { return this._peer;
return; //if (this._peers.length === 0) {
} // return;
var index = this._peerIndex++ % this._peers.length; //}
return this._peers[index]; //var index = this._peerIndex++ % this._peers.length;
//return this._peers[index];
}; };
P2P.prototype._hasPeers = function() { P2P.prototype._hasPeers = function() {
@ -275,7 +284,7 @@ P2P.prototype._onPeerReady = function(peer, addr) {
peer.port + ', best height: ' + peer.bestHeight); peer.port + ', best height: ' + peer.bestHeight);
this._addPeer(peer); this._addPeer(peer);
var bestHeight = this._getBestHeight(peer); var bestHeight = this._getBestHeight();
if (bestHeight >= 0) { if (bestHeight >= 0) {
this.emit('bestHeight', bestHeight); this.emit('bestHeight', bestHeight);