From 3bfd10d0d03586adb5d4d9b65e9f156d567b67d7 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 1 Dec 2015 22:59:26 -0800 Subject: [PATCH] work --- lib/bcoin/chain.js | 57 ++++++++++++++++++++- lib/bcoin/peer.js | 86 +++++++++++++++++++++++++++----- lib/bcoin/pool.js | 96 +++++++++++++++++++++++++++--------- lib/bcoin/protocol/parser.js | 4 +- lib/bcoin/tx.js | 4 ++ lib/bcoin/utils.js | 4 ++ lib/bcoin/wallet.js | 55 +++++++++++++++++++-- 7 files changed, 264 insertions(+), 42 deletions(-) diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 37cb4df3..130b234e 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -26,6 +26,7 @@ function Chain(options) { }; this.orphan = { map: {}, + bmap: {}, count: 0 }; this.index = { @@ -65,25 +66,35 @@ function compareTs(a, b) { } Chain.prototype._init = function _init() { + var self = this; + if (!this.storage) return; + utils.nextTick(function() { + self.emit('debug', 'Chain is loading.'); + }); + this.loading = true; - var self = this; + var s = this.storage.createReadStream({ start: this.prefix, end: this.prefix + 'z' }); + s.on('data', function(data) { var hash = data.key.slice(self.prefix.length); self._addIndex(hash, data.value.ts, data.value.height); }); + s.on('error', function(err) { self.emit('error', err); }); + s.on('end', function() { self.loading = false; self.emit('load'); + self.emit('debug', 'Chain successfully loaded.'); }); }; @@ -234,6 +245,7 @@ Chain.prototype.add = function add(block) { if (!this._probeIndex(hash, block.ts) && !prevProbe) { this.orphan.count++; this.orphan.map[prev] = block; + this.orphan.bmap[block.hash('hex')] = block; var range = this._getRange(hash, block.ts, true); var hashes = this.index.hashes.slice(range.start, range.end + 1); @@ -259,6 +271,7 @@ Chain.prototype.add = function add(block) { // We have orphan child for this block - add it to chain block = this.orphan.map[hash]; delete this.orphan.map[hash]; + delete this.orphan.bmap[block.hash('hex')]; this.orphan.count--; } while (true); @@ -414,6 +427,48 @@ Chain.prototype.getStartHeight = function getLast(cb) { } }; +Chain.prototype.getOrphanRoot = function getOrphanRoot(hash) { + var self = this; + + if (typeof hash !== 'string') { + hash = hash.hash('hex'); + } + + /* + var orphans = Object.keys(this.orphan.map).reduce(function(out, prev) { + var orphan = self.orphan.map[prev]; + out[orphan.hash('hex')] = orphan; + return out; + }, {}); + */ + var orphans = this.orphan.bmap; + + /* + var orphanRoot = hash; + var last = orphanRoot; + while (orphans[last]) { + orphanRoot = last; + last = orphans[last].prevBlock; + } + */ + + // accurate: + var orphanRoot = hash; + while (orphans[orphanRoot.prevBlock]) { + orphanRoot = orphans[orphanRoot.prevBlock]; + } + + /* + if hash stop gets last desired block, it should be: + var orphanRoot = hash; + while (orphans[orphanRoot]) { + orphanRoot = orphans[orphanRoot].prevBlock; + } + */ + + return orphanRoot; +}; + Chain.prototype.toJSON = function toJSON() { var keep = 1000; diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js index 66d8fa76..e2e7fb9b 100644 --- a/lib/bcoin/peer.js +++ b/lib/bcoin/peer.js @@ -93,7 +93,10 @@ Peer.prototype._init = function init() { this.once('version', function() { var ip = self.socket && self.socket.remoteAddress || '0.0.0.0'; self.pool.emit('debug', 'version (%s): loading locator hashes for getblocks', ip); - self.loadBlocks(self.pool.locatorHashes(), 0); + self.chain.on('load', function() { + // self.loadBlocks(self.pool.locatorHashes(), 0); + self.loadHeaders(self.pool.locatorHashes(), 0); + }); }); } @@ -288,6 +291,8 @@ Peer.prototype._onPacket = function onPacket(packet) { return this._handleVersion(payload); else if (cmd === 'inv') return this._handleInv(payload); + else if (cmd === 'headers') + return this._handleHeaders(payload); else if (cmd === 'getdata') return this._handleGetData(payload); else if (cmd === 'addr') @@ -299,12 +304,6 @@ Peer.prototype._onPacket = function onPacket(packet) { else if (cmd === 'getaddr') return this._handleGetAddr(); - if (cmd === 'headers') { - payload = bcoin.block(payload, 'block'); - this.emit(cmd, payload); - return; - } - if (cmd === 'merkleblock' || cmd === 'block') { payload = bcoin.block(payload, cmd); this.lastBlock = payload; @@ -439,6 +438,7 @@ Peer.prototype._handleInv = function handleInv(items) { }); this.emit('blocks', blocks); + if (0) if (this.pool.options.fullNode) { var self = this; @@ -449,24 +449,47 @@ Peer.prototype._handleInv = function handleInv(items) { this.getData(txs); } - var orphans = Object.keys(this.chain.orphan.map).reduce(function(out, prev) { + if (blocks.length === 1) { + this._rChainHead = utils.toHex(blocks[0]); + } + + var orph = blocks.filter(function(block) { + return self._requestingOrphan === utils.toHex(block); + }); + + if (orph.length) { + utils.debug('FOUND MISSING BLOCK'); + utils.debug('FOUND MISSING BLOCK'); + utils.debug('FOUND MISSING BLOCK'); + } + + /* + var orphans = Object.keys(self.chain.orphan.map).reduce(function(out, prev) { var orphan = self.chain.orphan.map[prev]; out[orphan.hash('hex')] = orphan; return out; }, {}); + */ + + var orphans = self.chain.orphan.bmap; for (var i = 0; i < blocks.length; i++) { var hash = utils.toHex(blocks[i]); if (orphans[hash]) { - this.loadBlocks(this.pool.locatorHashes(), orphans[hash].prevBlock); + this.loadBlocks(this.pool.locatorHashes(), this.chain.getOrphanRoot(hash)); continue; } - if (!this.chain.index.hashes[hash]) { - this.getData([{ type: 'block', hash: hash }]); - } else if (i === blocks.length - 1) { + if (!this.chain.index.bloom.test(hash, 'hex')) { this.getData([{ type: 'block', hash: hash }]); } } + +/* + this.getData(blocks.map(function(block) { + return { type: 'block', hash: utils.toHex(block) }; + })); +*/ + return; } @@ -479,6 +502,45 @@ Peer.prototype._handleInv = function handleInv(items) { this.getData(txs); }; +Peer.prototype._handleHeaders = function handleHeaders(headers) { + var self = this; + headers = headers.map(function(header) { + header.prevBlock = utils.toHex(header.prevBlock); + header.merkleRoot = utils.toHex(header.merkleRoot); + var abbr = bcoin.block.prototype.abbr.call(header); + header._hash = utils.toHex(utils.dsha256(abbr)); + return header; + }); + + if (this.pool.options.fullNode) { + var self = this; + + if (headers.length === 1) { + this._rChainHead = headers[0]._hash; + } + + var orphans = self.chain.orphan.bmap; + + for (var i = 0; i < headers.length; i++) { + var hash = headers[i]._hash; + if (orphans[hash]) { + this.loadHeaders(this.pool.locatorHashes(), this.chain.getOrphanRoot(hash)); + continue; + } + if (!this.chain.index.bloom.test(hash, 'hex')) { + this.getData([{ type: 'block', hash: hash }]); + } + } + + return; + } +}; + + +Peer.prototype.loadHeaders = function loadBlocks(hashes, stop) { + this._write(this.framer.getHeaders(hashes, stop)); +}; + Peer.prototype.loadBlocks = function loadBlocks(hashes, stop) { this._write(this.framer.getBlocks(hashes, stop)); }; diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index fc34bd2f..d09d85bd 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -96,18 +96,24 @@ inherits(Pool, EventEmitter); module.exports = Pool; Pool.prototype._init = function _init() { + var self = this; + this._addLoader(); for (var i = 0; i < this.size; i++) this._addPeer(0); this._load(); - var self = this; this.chain.on('missing', function(hash, preload, parent) { if (self.options.fullNode) return; self._request('block', hash, { force: true }); self._scheduleRequests(); self._loadRange(preload); }); + + this.chain.on('debug', function() { + var args = Array.prototype.slice.call(arguments); + self.emit.apply(self, ['debug'].concat(args)); + }); }; Pool.prototype._addLoader = function _addLoader() { @@ -148,6 +154,8 @@ Pool.prototype._addLoader = function _addLoader() { clearTimeout(timer); }); + if (this.options.fullNode) return; + function destroy() { // Chain is full and up-to-date if (self.chain.isFull()) { @@ -163,8 +171,6 @@ Pool.prototype._addLoader = function _addLoader() { // Split blocks and request them using multiple peers peer.on('blocks', function(hashes) { - if (self.options.fullNode) return; - if (hashes.length === 0) { // Reset global load self.block.lastHash = null; @@ -321,18 +327,26 @@ Pool.prototype._addPeer = function _addPeer(backoff) { peer.on('block', function(block) { backoff = 0; - // Ignore if we already have. - if (self.chain.index.hashes[block.hash('hex')] - || self.chain.orphan.map[block.prevBlock]) - return; - - // Store as orphan if prevBlock not in main chain - if (self.chain.index.hashes[block.prevBlock]) - peer.loadBlocks(self.locatorHashes(), block.prevBlock); + var hash = block.hash('hex'); + if (hash === peer._requestingOrphan) { + delete peer._requestingOrphan; + } + var height = self.chain.index.hashes.length; // Otherwise, accept block self._response(block); self.chain.add(block); + + if (self.chain.orphan.map[block.prevBlock] || !self.chain.index.bloom.test(block.prevBlock)) { + // if (!self.chain.index.bloom.test(block.prevBlock)) { + if (peer._requestingOrphan) return; + var orphanRoot = self.chain.getOrphanRoot(block); + peer._requestingOrphan = orphanRoot; + peer.loadHeaders(self.locatorHashes(), orphanRoot); + } else if (self.chain.index.hashes.length === height) { + return; + } + self.emit('chain-progress', self.chain.fillPercent(), peer); self.emit('block', block, peer); }); @@ -377,31 +391,65 @@ Pool.prototype._addPeer = function _addPeer(backoff) { }); }; -Pool.prototype.locatorHashes = function() { +Pool.prototype.locatorHashes = function(index) { + var chain = this.chain.index.hashes; + var hashes = []; + var top = chain.length - 1; + var step = 1; + var i; + + if (typeof index === 'string') { + for (i = top; i >= 0; i--) { + if (chain[i] === index) { + top = i; + break; + } + } + } else if (typeof index === 'number') { + top = index; + } + + i = top; + for (;;) { + hashes.push(chain[i]); + i = i - step; + if (i <= 0) { + hashes.push(chain[0]); + break; + } + if (hashes.length >= 10) { + step *= 2; + } + } + + return hashes; +}; + +Pool.prototype.locatorHashes = function(index) { var self = this; var hashes = this.chain.index.hashes; var indicies = []; var top = hashes.length - 1; - var step = 1; + var step = 1, start = 0; - for (var j = 0, i = top - 1; j < 10 && i > 0; j++, i--) { - indicies.push(hashes[i]); + if (typeof index === 'string') { + for (i = top; i >= 0; i--) { + if (chain[i] === index) { + top = i; + break; + } + } + } else if (typeof index === 'number') { + top = index; } - for (; i > 0; i -= step) { + for (var i = top; i > 0; i -= step, ++start) { + if (start >= 10) step *= 2; indicies.push(hashes[i]); - step *= 2; } indicies.push(hashes[0]); - indicies = indicies.reduce(function(out, hash) { - if (!~out.indexOf(hash)) { - out.push(hash); - } - return out; - }, []); - return indicies; }; diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index 3a2b1da0..39ecf24d 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -235,7 +235,7 @@ Parser.prototype.parseHeaders = function parseHeaders(p) { var headers = []; if (p.length >= off + 81) { - for (var i = 0; i < count; i++) { + for (var i = 0; i < count && off + 81 < p.length; i++) { var header = {}; header.version = readU32(p, off); off += 4; @@ -251,7 +251,7 @@ Parser.prototype.parseHeaders = function parseHeaders(p) { off += 4; var r = readIntv(p, off); header.totalTX = r.r; - off += r.off; + off = r.off; headers.push(header); } } diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index 7b15e799..970784d0 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -300,3 +300,7 @@ TX.fromJSON = function fromJSON(json) { return tx; }; + +TX.prototype.clone = function clone() { + return new TX(new bcoin.protocol.parser().parseTX(this.render())); +}; diff --git a/lib/bcoin/utils.js b/lib/bcoin/utils.js index 44ac8387..748193c1 100644 --- a/lib/bcoin/utils.js +++ b/lib/bcoin/utils.js @@ -466,3 +466,7 @@ utils.isIP = function(ip) { return 0; }; + +utils.debug = function(msg) { + console.log('\x1b[31m' + msg + '\x1b[m'); +}; diff --git a/lib/bcoin/wallet.js b/lib/bcoin/wallet.js index 877f7a44..65b64919 100644 --- a/lib/bcoin/wallet.js +++ b/lib/bcoin/wallet.js @@ -250,6 +250,33 @@ Wallet.prototype.sign = function sign(tx, type, inputs, off) { return inputs.length; }; +Wallet.prototype.signEmpty = function sign(tx, type, inputs, off) { + if (!type) + type = 'all'; + assert.equal(type, 'all'); + + if (!off) + off = 0; + + var pub = this.getPublicKey(); + inputs = inputs || tx.inputs; + + // Add signature script to each input + inputs = inputs.filter(function(input, i) { + // Filter inputs that this wallet own + if (!input.out.tx || !this.ownOutput(input.out.tx)) + return false; + + var signature = [0x30, 0, 0x02, 0, 0, 0x02, 0, 0]; + signature = signature.concat(bcoin.protocol.constants.hashType[type]); + + input.script = [ signature, pub ]; + return true; + }, this); + + return inputs.length; +}; + Wallet.prototype.addTX = function addTX(tx, block) { return this.tx.add(tx); }; @@ -270,9 +297,17 @@ Wallet.prototype.balance = function balance() { return this.tx.balance(); }; -Wallet.prototype.fill = function fill(tx, cb) { +Wallet.prototype.fill = function fill(tx, options, cb) { + if ((cb && typeof cb === 'object') || options == null) { + cb = options; + options = {}; + } cb = utils.asyncify(cb); + if (options._getChange) { + tx = tx.clone(); + } + // NOTE: tx should be prefilled with all outputs var cost = tx.funds('out'); @@ -296,7 +331,7 @@ Wallet.prototype.fill = function fill(tx, cb) { unspent.every(addInput, this); // Add dummy output (for `left`) to calculate maximum TX size - tx.out(this, new bn(0)); + tx.out(options.change || this, new bn(0)); // Change fee value if it is more than 1024 bytes // (10000 satoshi for every 1024 bytes) @@ -323,6 +358,10 @@ Wallet.prototype.fill = function fill(tx, cb) { // How much money is left after sending outputs var left = tx.funds('in').sub(total); + if (options._getChange) { + return left; + } + // Not enough money, transfer everything to owner if (left.cmpn(this.dust) < 0) { // NOTE: that this output is either `postCost` or one of the `dust` values @@ -337,9 +376,19 @@ Wallet.prototype.fill = function fill(tx, cb) { tx.outputs[tx.outputs.length - 1].value = left; // Sign transaction - this.sign(tx); + if (options.sign === false) { + this.signEmpty(tx); + } else { + this.sign(tx); + } cb(null, tx); + + return tx; +}; + +Wallet.prototype.getChange = function fill(tx) { + return this.fill(tx, { _getChange: true }); }; Wallet.prototype.toJSON = function toJSON() {