From 6b45ef27cd3d93e00a73723aaeb36a7e6642f737 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 19 Jul 2017 18:58:18 -0400 Subject: [PATCH] wip --- lib/services/block/encoding.js | 29 +------ lib/services/block/index.js | 148 +++++--------------------------- lib/services/header/encoding.js | 21 +++-- lib/services/header/index.js | 100 ++++++++++++++++----- lib/services/p2p/index.js | 35 +++++--- 5 files changed, 142 insertions(+), 191 deletions(-) diff --git a/lib/services/block/encoding.js b/lib/services/block/encoding.js index 3987dda8..fe82c6a8 100644 --- a/lib/services/block/encoding.js +++ b/lib/services/block/encoding.js @@ -5,18 +5,16 @@ var Block = require('bitcore-lib').Block; function Encoding(servicePrefix) { this._servicePrefix = servicePrefix; - this._blockPrefix = new Buffer('00', 'hex'); - this._metaPrefix = new Buffer('01', 'hex'); } // ---- hash --> rawblock Encoding.prototype.encodeBlockKey = function(hash) { - return Buffer.concat([ this._servicePrefix, this._blockPrefix, new Buffer(hash, 'hex') ]); + return Buffer.concat([ this._servicePrefix, new Buffer(hash, 'hex') ]); }; Encoding.prototype.decodeBlockKey = function(buffer) { - return buffer.slice(3).toString('hex'); + return buffer.slice(2).toString('hex'); }; Encoding.prototype.encodeBlockValue = function(block) { @@ -27,27 +25,4 @@ Encoding.prototype.decodeBlockValue = function(buffer) { return Block.fromBuffer(buffer); }; -// ---- height --> hash, chainwork -Encoding.prototype.encodeMetaKey = function(height) { - var heightBuf = new Buffer(4); - heightBuf.writeUInt32BE(height); - return Buffer.concat([ this._servicePrefix, this._metaPrefix, heightBuf ]); -}; - -Encoding.prototype.decodeMetaKey = function(buffer) { - return buffer.readUInt32BE(3); -}; - -Encoding.prototype.encodeMetaValue = function(value) { - // { chainwork: hex-string, hash: hex-string } - return Buffer.concat([ new Buffer(value.hash, 'hex'), new Buffer(value.chainwork, 'hex') ]); -}; - -Encoding.prototype.decodeMetaValue = function(buffer) { - return { - hash: buffer.slice(0, 32).toString('hex'), - chainwork: buffer.slice(32).toString('hex') - }; -}; - module.exports = Encoding; diff --git a/lib/services/block/index.js b/lib/services/block/index.js index f0a6e00a..8b0489d6 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -11,7 +11,6 @@ var utils = require('../../utils'); var _ = require('lodash'); var assert = require('assert'); var BN = require('bn.js'); -var consensus = require('bcoin').consensus; var constants = require('../../constants'); var BlockService = function(options) { @@ -21,18 +20,12 @@ var BlockService = function(options) { this._tip = null; this._p2p = this.node.services.p2p; this._db = this.node.services.db; + this._header = this.node.services.header; this._subscriptions = {}; this._subscriptions.block = []; this._subscriptions.reorg = []; - //memory - this._maxMem = options.maxMemory || 1500; // in MB - this._deferTimeout = null; - - // meta is [{ chainwork: chainwork, hash: hash }] - this._meta = []; - // in-memory full/raw block cache this._blockQueue = LRU({ max: 50 * (1 * 1024 * 1024), // 50 MB of blocks, @@ -53,9 +46,8 @@ var BlockService = function(options) { inherits(BlockService, BaseService); -BlockService.dependencies = [ 'p2p', 'db' ]; +BlockService.dependencies = [ 'p2p', 'db', 'header' ]; -BlockService.MAX_CHAINWORK = new BN(1).ushln(256); BlockService.MAX_BLOCKS = 500; // --- public prototype functions @@ -71,7 +63,8 @@ BlockService.prototype.getAPIMethods = function() { }; BlockService.prototype.getBestBlockHash = function(callback) { - return callback(null, this._meta[this._meta.length - 1].hash); + var headers = this._header.getAllHeaders(); + return headers[headers.length - 1].hash; }; BlockService.prototype.getBlock = function(arg, callback) { @@ -95,6 +88,7 @@ BlockService.prototype.getBlockHeader = function(blockArg, callback) { return callback(); } + var headers = this._header.getAllHeaders(); this._getBlock(blockArg, function(err, block) { if(err) { @@ -188,15 +182,17 @@ BlockService.prototype.start = function(callback) { function(tip, next) { self._tip = tip; self._chainTips.push(self._tip.hash); - self._loadMeta(next); } ], function(err) { + if(err) { return callback(err); } + self._setListeners(); self._startSubscriptions(); callback(); + }); }; @@ -270,24 +266,6 @@ BlockService.prototype._cacheBlock = function(block) { }; -BlockService.prototype._computeChainwork = function(bits, prev) { - - var target = consensus.fromCompact(bits); - - if (target.isNeg() || target.cmpn(0) === 0) { - return new BN(0); - } - - var proof = BlockService.MAX_CHAINWORK.div(target.iaddn(1)); - - if (!prev) { - return proof; - } - - return proof.iadd(prev); - -}; - BlockService.prototype._determineBlockState = function(block) { if (this._isOutOfOrder(block)) { @@ -365,16 +343,6 @@ BlockService.prototype._getBlockOperations = function(block) { }; -BlockService.prototype._getChainwork = function(tipHash) { - - var block = this._blockQueue.get(tipHash); - - var lastChainwork = this._meta[this._meta.length - 1].chainwork; - var prevChainwork = new BN(new Buffer(lastChainwork, 'hex')); - - return this._computeChainwork(block.header.bits, prevChainwork); -}; - BlockService.prototype._getDelta = function(tip) { var blocks = []; @@ -393,8 +361,9 @@ BlockService.prototype._getDelta = function(tip) { BlockService.prototype._getHash = function(blockArg) { + var headers = this._header.getAllHeaders(); return (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) && - this._meta[blockArg] ? this._meta[blockArg] : null; + headers[blockArg] ? headers[blockArg] : null; }; @@ -411,15 +380,15 @@ BlockService.prototype._getIncompleteChainIndexes = function(block) { }; BlockService.prototype._getOldBlocks = function(currentHash, commonAncestorHash) { - // the old blocks should be in the meta colection if (currentHash === commonAncestorHash || !commonAncestorHash || !currentHash) { return; } var oldBlocks; - for(var i = this._meta.length - 1; i > 0; --i) { - var item = this._meta[i]; + var headers = this._header.getAllHeaders(); + for(var i = headers.length - 1; i > 0; --i) { + var item = headers[i]; if (item.hash === currentHash) { oldBlocks = [this._blockQueue.get(currentHash)]; continue; @@ -472,28 +441,14 @@ BlockService.prototype._onReorg = function(oldBlockList, newBlockList, commonAnc // remove all the old blocks that we reorg from oldBlockList.forEach(function(block) { - removalOps.concat([ - { - type: 'del', - key: this.encoding.encodeBlockKey(block.header.timestamp), - }, - { - type: 'del', - key: this.encoding.encodeMetaKey(block.header.height), - } - ]); + removalOps.push({ + type: 'del', + key: this.encoding.encodeBlockKey(block.header.timestamp), + }); }); this._db.batch(removalOps); - // remove the blocks from the in-memory meta list - var newMetaHeight = commonAncestor.header.height; - for(var i = this._meta.length - 1; i > 0; --i) { - if (i > newMetaHeight) { - this._meta.splice(i, 1); - } - } - //call onBlock for each of the new blocks newBlockList.forEach(this._onBlock.bind(this)); // if the common ancestor block height is greater than our own, then nothing to do for the reorg @@ -528,32 +483,8 @@ BlockService.prototype._isOutOfOrder = function(block) { }; -BlockService.prototype._loadMeta = function(callback) { - var self = this; - var criteria = { - gte: self._encoding.encodeMetaKey(0), - lte: self._encoding.encodeMetaKey(0xffffffff) - }; - - var stream = this._db.createReadStream(criteria); - - stream.on('end', function() { - if (self._meta.length < 1) { - self._meta.push({ - chainwork: '0000000000000000000000000000000000000000000000000000000100010001', - hash: self.GENESIS_HASH - }); - } - callback(); - }); - - stream.on('data', function(data) { - self._meta.push(self._encoding.decodeMetaValue(data.value)); - }); -}; - -BlockService.prototype._onBestHeight = function(height) { - this._bestHeight = height; +BlockService.prototype._onAllHeaders = function(headers) { + this._bestHeight = headers.length; this._startSync(); }; @@ -591,45 +522,11 @@ BlockService.prototype._onBlock = function(block) { break; default: // send all unsent blocks now that we have a complete chain - this._saveMetaData(block); this._sendDelta(); break; } }; -BlockService.prototype._saveMetaData = function(block) { - var item = { - chainwork: this._getChainwork(block.hash).toString(16, 64), - hash: block.hash - }; - - this._meta.push(item); - - var operations = []; - - // tip - this._tip.hash = block.hash; - this._tip.height = this._meta.length; - - var tipInfo = utils.encodeTip(this._tip, this.name); - - operations.push({ - type: 'put', - key: tipInfo.key, - value: tipInfo.value - }); - - //meta positions in db - operations.push({ - type: 'put', - key: this._encoding.encodeMetaKey(this._meta.length - 1), - value: this._encoding.encodeMetaValue(item) - }); - - this._db.batch(operations); -}; - - BlockService.prototype._selectActiveChain = function() { var chainTip; @@ -664,8 +561,9 @@ BlockService.prototype._sendDelta = function() { this._broadcast(this._subscriptions.block, 'block/block', blocks[i]); } - var len = this._meta.length - 1; - this._setTip({ height: len, hash: this._meta[len].hash }); + var headers = this._header.getAllHeaders(); + var len = headers.length - 1; + this._setTip({ height: len, hash: headers[len].hash }); if (++this._blockCount >= BlockService.MAX_BLOCKS) { this._latestBlockHash = this._tip.hash; @@ -710,7 +608,7 @@ BlockService.prototype._continueSync = function() { BlockService.prototype._setListeners = function() { - this._p2p.once('bestHeight', this._onBestHeight.bind(this)); + this._header.once('headers', this._onAllHeaders.bind(this)); }; diff --git a/lib/services/header/encoding.js b/lib/services/header/encoding.js index 348bbb19..9bcdb69b 100644 --- a/lib/services/header/encoding.js +++ b/lib/services/header/encoding.js @@ -7,12 +7,20 @@ function Encoding(servicePrefix) { // ---- hash --> header -Encoding.prototype.encodeHeaderKey = function(hash) { - return Buffer.concat([ this._servicePrefix, new Buffer(hash, 'hex') ]); +Encoding.prototype.encodeHeaderKey = function(height, hash) { + var heightBuf = new Buffer(4); + heightBuf.writeUInt32BE(height); + var hashBuf = new Buffer(hash || new Array(65).join('0'), 'hex'); + return Buffer.concat([ this._servicePrefix, heightBuf, hashBuf ]); }; Encoding.prototype.decodeHeaderKey = function(buffer) { - return buffer.slice(2).toString('hex'); + var height = buffer.readUInt32BE(2); + return { + height: height, + hash: buffer.slice(6).toString('hex') + }; + }; Encoding.prototype.encodeHeaderValue = function(header) { @@ -28,7 +36,8 @@ Encoding.prototype.encodeHeaderValue = function(header) { nonceBuf.writeUInt32BE(header.nonce); var heightBuf = new Buffer(4); heightBuf.writeUInt32BE(header.height); - return Buffer.concat([ versionBuf, prevHash, merkleRoot, tsBuf, bitsBuf, nonceBuf, heightBuf ]); + var chainworkBuf = new Buffer(header.chainwork, 'hex'); + return Buffer.concat([ versionBuf, prevHash.reverse(), merkleRoot.reverse(), tsBuf, bitsBuf, nonceBuf, heightBuf, chainworkBuf ]); }; Encoding.prototype.decodeHeaderValue = function(buffer) { @@ -39,6 +48,7 @@ Encoding.prototype.decodeHeaderValue = function(buffer) { var bits = buffer.readUInt32BE(72); var nonce = buffer.readUInt32BE(76); var height = buffer.readUInt32BE(80); + var chainwork = buffer.slice(84).toString('hex'); return { version: version, prevHash: prevHash, @@ -46,7 +56,8 @@ Encoding.prototype.decodeHeaderValue = function(buffer) { timestamp: ts, bits: bits, nonce: nonce, - height: height + height: height, + chainwork: chainwork }; }; diff --git a/lib/services/header/index.js b/lib/services/header/index.js index 5c82d06f..0ae2b18c 100644 --- a/lib/services/header/index.js +++ b/lib/services/header/index.js @@ -6,6 +6,9 @@ var Encoding = require('./encoding'); var index = require('../../'); var log = index.log; var utils = require('../../utils'); +var async = require('async'); +var BN = require('bn.js'); +var consensus = require('bcoin').consensus; var HeaderService = function(options) { @@ -14,48 +17,61 @@ var HeaderService = function(options) { this._tip = null; this._p2p = this.node.services.p2p; this._db = this.node.services.db; + this._headers = []; }; inherits(HeaderService, BaseService); HeaderService.dependencies = [ 'p2p', 'db' ]; +HeaderService.MAX_CHAINWORK = new BN(1).ushln(256); +HeaderService.STARTING_CHAINWORK = '0000000000000000000000000000000000000000000000000000000100010001'; // --- public prototype functions HeaderService.prototype.getAPIMethods = function() { var methods = [ - ['getAllHeaders', this, this.getAllHeaders, 0] + ['getAllHeaders', this, this.getAllHeaders, 0], + ['getBestHeight', this, this.getBestHeight, 0] ]; return methods; }; +HeaderService.prototype.getBestHeight = function() { + return this._tip.height; +}; + HeaderService.prototype.start = function(callback) { var self = this; - self._db.getPrefix(self.name, function(err, prefix) { + async.waterfall([ + function(next) { + self._db.getPrefix(self.name, next); + }, + function(prefix, next) { + self._encoding = new Encoding(prefix); + self._db.getServiceTip(self.name, next); + }, + function(tip, next) { + self._tip = tip; + self._getPersistedHeaders(next); + } + ], function(err, headers) { - if(err) { + if (err) { return callback(err); } - self._db.getServiceTip(self.name, function(err, tip) { + this._headers = headers; + self._setListeners(); + self._startSubscriptions(); + callback(); - if(err) { - return callback(err); - } - - self._tip = tip; - self._encoding = new Encoding(prefix); - self._setListeners(); - self._startSubscriptions(); - callback(); - - }); }); + }; HeaderService.prototype.stop = function(callback) { @@ -74,7 +90,15 @@ HeaderService.prototype._startSubscriptions = function() { } this._bus.on('p2p/headers', this._onHeaders.bind(this)); + this._bus.on('p2p/block', this._onHeaders.bind(this)); this._bus.subscribe('p2p/headers'); + this._bus.subscribe('p2p/block'); + +}; + +HeaderService.prototype._onBlock = function(block) { + // we just want the header to keep a running list + this._onHeaders([block.header]); }; HeaderService.prototype._onHeaders = function(headers) { @@ -83,11 +107,11 @@ HeaderService.prototype._onHeaders = function(headers) { return; } + var operations = this._getHeaderOperations(headers); + this._tip.hash = headers[headers.length - 1].hash; this._tip.height = this._tip.height + headers.length; - var operations = this._getHeaderOperations(headers); - var tipOps = utils.encodeTip(this._tip, this.name); operations.push({ @@ -97,10 +121,11 @@ HeaderService.prototype._onHeaders = function(headers) { }); this._db.batch(operations); + this._headers.concat(headers); if (this._tip.height >= this._bestHeight) { log.info('Header download complete.'); - this.emit('headers'); + this.emit('headers', this._headers); return; } @@ -112,11 +137,14 @@ HeaderService.prototype._getHeaderOperations = function(headers) { var self = this; var runningHeight = this._tip.height; + var prevHeader = this._headers[this._headers.length - 1]; return headers.map(function(header) { header.height = ++runningHeight; + header.chainwork = self._getChainwork(header, prevHeader).toString(16, 32); + prevHeader = header; return { type: 'put', - key: self._encoding.encodeHeaderKey(header.hash), + key: self._encoding.encodeHeaderKey(header.height, header.hash), value: self._encoding.encodeHeaderValue(header) }; }); @@ -163,7 +191,11 @@ HeaderService.prototype._sync = function() { }; -HeaderService.prototype.getAllHeaders = function(callback) { +HeaderService.prototype.getAllHeaders = function() { + return this._headers; +}; + +HeaderService.prototype._getPersistedHeaders = function(callback) { var self = this; var results = []; @@ -183,7 +215,7 @@ HeaderService.prototype.getAllHeaders = function(callback) { stream.on('data', function(data) { var res = {}; - res[self._encoding.decodeHeaderKey(data.key)] = self._encoding.decodeHeaderValue(data.value); + res[self._encoding.decodeHeaderKey(data.key).hash] = self._encoding.decodeHeaderValue(data.value); results.push(res); }); @@ -196,5 +228,31 @@ HeaderService.prototype.getAllHeaders = function(callback) { }; +HeaderService.prototype._getChainwork = function(header, prevHeader) { + + var lastChainwork = prevHeader ? prevHeader.chainwork : HeaderService.STARTING_CHAINWORK; + var prevChainwork = new BN(new Buffer(lastChainwork, 'hex')); + + return this._computeChainwork(header.bits, prevChainwork); +}; + +HeaderService.prototype._computeChainwork = function(bits, prev) { + + var target = consensus.fromCompact(bits); + + if (target.isNeg() || target.cmpn(0) === 0) { + return new BN(0); + } + + var proof = HeaderService.MAX_CHAINWORK.div(target.iaddn(1)); + + if (!prev) { + return proof; + } + + return proof.iadd(prev); + +}; + module.exports = HeaderService; diff --git a/lib/services/p2p/index.js b/lib/services/p2p/index.js index 2c57becf..301ac0e6 100644 --- a/lib/services/p2p/index.js +++ b/lib/services/p2p/index.js @@ -22,7 +22,7 @@ var P2P = function(options) { this._initP2P(); this._initPubSub(); this._bcoin = null; - + this._currentBestHeight = null; }; util.inherits(P2P, BaseService); @@ -41,6 +41,7 @@ P2P.prototype.getAPIMethods = function() { ['getInfo', this, this.getInfo, 0], ['getMempool', this, this.getMempool, 0], ['sendTransaction', this, this.sendTransaction, 1] + ['getBestHeight', this, this.getBestHeight, 0] ]; return methods; }; @@ -171,23 +172,31 @@ P2P.prototype._connect = function() { } }; -P2P.prototype._getBestHeight = function(peer) { - - this._peerHeights.push(peer.bestHeight); - - if (this._peerHeights.length >= this._minPeers) { - return Math.max(...this._peerHeights); +P2P.prototype._getBestHeight = function() { + if (this._peers < this._minPeers) { + return 0; } + var maxHeight = 0; + for(var i = 0; i < this._peers.length; i++) { + if (this._peers[i].bestHeight > maxHeight) { + maxHeight = this._peers[i].bestHeight; + this._peer = this._peers[i]; + } + } + return maxHeight; + }; +// we should only choose from a list of peers that sync'ed P2P.prototype._getPeer = function() { - if (this._peers.length === 0) { - return; - } - var index = this._peerIndex++ % this._peers.length; - return this._peers[index]; + return this._peer; + //if (this._peers.length === 0) { + // return; + //} + //var index = this._peerIndex++ % this._peers.length; + //return this._peers[index]; }; P2P.prototype._hasPeers = function() { @@ -275,7 +284,7 @@ P2P.prototype._onPeerReady = function(peer, addr) { peer.port + ', best height: ' + peer.bestHeight); this._addPeer(peer); - var bestHeight = this._getBestHeight(peer); + var bestHeight = this._getBestHeight(); if (bestHeight >= 0) { this.emit('bestHeight', bestHeight);