diff --git a/lib/bcoin/block.js b/lib/bcoin/block.js index 9fc85d98..9716938f 100644 --- a/lib/bcoin/block.js +++ b/lib/bcoin/block.js @@ -46,58 +46,47 @@ Block.prototype.render = function render() { }; Block.prototype.renderNormal = function renderNormal() { - this.getRaw(); - - if (!this._witnessSize) - return this._raw; - return bcoin.protocol.framer.block(this); }; Block.prototype.renderWitness = function renderWitness() { - this.getRaw(); - - if (this._witnessSize) - return this._raw; - return bcoin.protocol.framer.witnessBlock(this); }; Block.prototype.getRaw = function getRaw() { var raw; - if (this._raw) { - assert(this._size > 0); - assert(this._witnessSize >= 0); - return this._raw; - } - if (this.hasWitness()) raw = bcoin.protocol.framer.witnessBlock(this); else raw = bcoin.protocol.framer.block(this); - this._raw = raw; this._size = raw.length; this._witnessSize = raw._witnessSize; - return this._raw; + return raw; }; Block.prototype.getVirtualSize = function getVirtualSize() { var size, witnessSize, base; - this.getRaw(); - - size = this._size; - witnessSize = this._witnessSize; + size = this.getSize(); + witnessSize = this.getWitnessSize(); base = size - witnessSize; return (base * 4 + witnessSize + 3) / 4 | 0; }; Block.prototype.getSize = function getSize() { - return this.getRaw().length; + if (this._size == null) + this.getRaw(); + return this._size; +}; + +Block.prototype.getWitnessSize = function getWitnessSize() { + if (this._witnessSize == null) + this.getRaw(); + return this._witnessSize; }; Block.prototype.hasWitness = function hasWitness() { @@ -272,8 +261,6 @@ Block.prototype.inspect = function inspect() { type: this.type, height: this.height, hash: utils.revHex(this.hash('hex')), - reward: utils.btc(this.getReward()), - fee: utils.btc(this.getFee()), date: new Date(this.ts * 1000).toISOString(), version: this.version, prevBlock: utils.revHex(this.prevBlock), diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js index 9acfbeab..bc4be15b 100644 --- a/lib/bcoin/blockdb.js +++ b/lib/bcoin/blockdb.js @@ -39,6 +39,7 @@ function BlockDB(node, options) { this.file = bcoin.prefix + '/block-' + network.type + '.db'; this.options = options; + this.fsync = !!options.fsync; this.node = node; @@ -134,7 +135,7 @@ BlockDB.prototype.migrate = function migrate(blockSize, compression, callback) { BlockDB.prototype.saveBlock = function saveBlock(block, callback) { var self = this; - var batch = this.db.batch(); + var batch = this.batch(); batch.put('b/b/' + block.hash('hex'), block.toCompact()); @@ -148,7 +149,7 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) { BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { var self = this; - this._getCoinBlock(hash, function(err, block) { + this._getTXBlock(hash, function(err, block) { var batch; if (err) @@ -157,22 +158,15 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { if (!block) return callback(); - batch = self.db.batch(); + batch = self.batch(); - function cb(err) { - if (err) - return callback(err); + batch.del('b/b/' + block.hash('hex')); - batch.del('b/b/' + block.hash('hex')); + block.txs.forEach(function(tx, i) { + batch.del('t/t/' + tx.hash('hex')); + }); - block.txs.forEach(function(tx, i) { - batch.del('t/t/' + tx.hash('hex')); - }); - - return callback(null, block); - } - - self.disconnectBlock(hash, cb, batch); + self.disconnectBlock(block, callback, batch); }); }; @@ -189,7 +183,7 @@ BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) { } if (!batch) - batch = self.db.batch(); + batch = self.batch(); batch.put('b/h/' + pad32(block.height), block.hash()); @@ -258,9 +252,7 @@ BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) { BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, batch) { var self = this; - this._getCoinBlock(hash, function(err, block) { - var batch; - + this._getTXBlock(hash, function(err, block) { if (err) return callback(err); @@ -270,7 +262,7 @@ BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, bat } if (!batch) - batch = self.db.batch(); + batch = self.batch(); if (typeof hash === 'string') assert(block.hash('hex') === hash); @@ -421,9 +413,8 @@ BlockDB.prototype.fillTX = function fillTX(tx, callback) { if (err) return next(err); - if (tx) { + if (tx) input.output = bcoin.coin(tx, input.prevout.index); - } next(); }); @@ -699,6 +690,23 @@ BlockDB.prototype._getCoinBlock = function _getCoinBlock(hash, callback) { }); }; +BlockDB.prototype._getTXBlock = function _getTXBlock(hash, callback) { + var self = this; + + if (hash instanceof bcoin.block) + return callback(null, hash); + + return this.getBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + return self.fillTXBlock(block, callback); + }); +}; + BlockDB.prototype.fillBlock = function fillBlock(block, callback) { var self = this; @@ -731,6 +739,38 @@ BlockDB.prototype.fillBlock = function fillBlock(block, callback) { }); }; +BlockDB.prototype.fillTXBlock = function fillTXBlock(block, callback) { + var self = this; + + return this.fillTXs(block.txs, function(err) { + var coins, i, tx, hash, j, input, id; + + if (err) + return callback(err); + + coins = {}; + + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + hash = tx.hash('hex'); + + for (j = 0; j < tx.inputs.length; j++) { + input = tx.inputs[j]; + id = input.prevout.hash + '/' + input.prevout.index; + if (!input.output && coins[id]) { + input.output = coins[id]; + delete coins[id]; + } + } + + for (j = 0; j < tx.outputs.length; j++) + coins[hash + '/' + j] = bcoin.coin(tx, j); + } + + return callback(null, block); + }); +}; + BlockDB.prototype._getHash = function _getHash(height, callback) { if (typeof height === 'string') return callback(null, height); @@ -778,7 +818,7 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) { block.txs[i] = tx; next(); - }) + }); }, function(err) { if (err) return callback(err); @@ -975,6 +1015,12 @@ BlockDB.prototype.reset = function reset(height, callback, emit) { }); }; +BlockDB.prototype.batch = function batch() { + if (this.fsync) + return new utils.SyncBatch(this.db); + return this.db.batch(); +}; + /** * Expose */ diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 3690a3b5..36373803 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -162,6 +162,15 @@ Chain.prototype._init = function _init() { self.tip = tip; self.height = tip.height; + + // Start fsyncing writes once we're no + // longer dealing with historical data. + if (tip.height >= network.checkpoints.lastHeight) { + self.db.fsync = true; + if (self.blockdb) + self.blockdb.fsync = true; + } + self.loading = false; self.emit('load'); }); @@ -490,8 +499,6 @@ Chain.prototype._verify = function _verify(block, prev, callback) { if (block.bits !== self.getTarget(prev, block)) { utils.debug('Block is using wrong target: %s', block.rhash); - utils.debug('Has %d, expected: %d', - block.bits, self.getTarget(prev, block)); return done(null, false); } @@ -982,6 +989,14 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) { self.tip = entry; self.height = entry.height; + // Start fsyncing writes once we're no + // longer dealing with historical data. + if (entry.height >= network.checkpoints.lastHeight) { + self.db.fsync = true; + if (self.blockdb) + self.blockdb.fsync = true; + } + return callback(); }); }); @@ -1025,7 +1040,7 @@ Chain.prototype.reset = function reset(height, callback, force) { self.emit('remove block', block); }); }, function(entry) { - self.emit('remove entry', block); + self.emit('remove entry', entry); }); }; @@ -1091,25 +1106,24 @@ Chain.prototype.resetTime = function resetTime(ts, callback, force) { if (!unlock) return; + callback = utils.ensure(callback); + this.byTime(ts, function(err, entry) { if (err) { unlock(); - if (callback) - callback(err); + callback(err); return; } if (!entry) { unlock(); - if (callback) - callback(); + callback(); return; } self.reset(entry.height, function(err) { unlock(); - if (callback) - callback(err); + callback(err); }, true); }, true); }; diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index 178c25a4..cf44f5ca 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -47,6 +47,7 @@ function ChainDB(node, chain, options) { this.fd = null; this.loading = false; this.loaded = false; + this.fsync = !!options.fsync; // Keep track of block hashes in a // bloom filter to avoid DB lookups. @@ -64,8 +65,8 @@ function ChainDB(node, chain, options) { else this._cacheWindow = network.block.majorityWindow + 1; - this.cacheHash = new DumbCache(this._cacheWindow * 200); // (not hashcash) - this.cacheHeight = new DumbCache(this._cacheWindow * 200); + this.cacheHash = new NullCache(this._cacheWindow * 200); // (not hashcash) + this.cacheHeight = new NullCache(this._cacheWindow * 200); // this.cacheHash = new bcoin.lru(this._cacheWindow, function() { return 1; }); // (not hashcash) // this.cacheHeight = new bcoin.lru(this._cacheWindow, function() { return 1; }); @@ -370,7 +371,7 @@ ChainDB.prototype.save = function save(entry, callback) { // this.bloom.add(entry.hash, 'hex'); - batch = this.db.batch(); + batch = this.batch(); height = new Buffer(4); utils.writeU32(height, entry.height, 0); @@ -412,7 +413,7 @@ ChainDB.prototype.connect = function connect(block, callback, emit) { if (!entry) return callback(); - batch = self.db.batch(); + batch = self.batch(); batch.put('c/h/' + pad32(entry.height), new Buffer(entry.hash, 'hex')); batch.put('c/t', new Buffer(entry.hash, 'hex')); @@ -438,7 +439,7 @@ ChainDB.prototype.disconnect = function disconnect(block, callback) { if (!entry) return callback(); - batch = self.db.batch(); + batch = self.batch(); batch.del('c/h/' + pad32(entry.height)); batch.put('c/t', new Buffer(entry.prevBlock, 'hex')); @@ -489,7 +490,7 @@ ChainDB.prototype.reset = function reset(block, callback, emit) { if (!tip) return callback(); - batch = self.db.batch(); + batch = self.batch(); (function next(err, tip) { if (err) @@ -528,6 +529,12 @@ ChainDB.prototype.has = function has(height, callback) { }); }; +ChainDB.prototype.batch = function batch() { + if (this.fsync) + return new utils.SyncBatch(this.db); + return this.db.batch(); +}; + function DumbCache(size) { this.data = {}; this.count = 0; @@ -573,6 +580,14 @@ DumbCache.prototype.reset = function reset() { this.count = 0; }; +function NullCache(size) {} + +NullCache.prototype.set = function set(key, value) {}; +NullCache.prototype.remove = function remove(key) {}; +NullCache.prototype.get = function get(key) {}; +NullCache.prototype.has = function has(key) {}; +NullCache.prototype.reset = function reset() {}; + /** * Expose */ diff --git a/lib/bcoin/compactblock.js b/lib/bcoin/compactblock.js index dbc5d11c..f5192696 100644 --- a/lib/bcoin/compactblock.js +++ b/lib/bcoin/compactblock.js @@ -38,7 +38,10 @@ CompactBlock.prototype.getCoinbaseHeight = function getCoinbaseHeight() { }; CompactBlock.prototype.toBlock = function toBlock() { - var block = new bcoin.block(bcoin.protocol.parser.parseBlock(this._raw)); + var block = bcoin.protocol.parser.parseBlock(this._raw); + delete this._raw; + assert(!block._raw); + block = new bcoin.block(block); if (this.valid != null) block.valid = this.valid; return block; diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index dc9b1e53..0746a27b 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -77,7 +77,7 @@ Mempool.prototype.removeBlock = function removeBlock(block) { self.removeTX(mtx); }); // Add transaction back into mempool - tx = tx.clone(); + // tx = tx.clone(); tx.ps = utils.now(); tx.ts = 0; tx.block = null; diff --git a/lib/bcoin/node.js b/lib/bcoin/node.js index fc7ddfe1..3fc7ac70 100644 --- a/lib/bcoin/node.js +++ b/lib/bcoin/node.js @@ -47,7 +47,8 @@ Fullnode.prototype._init = function _init() { // chain, but that's only once it's being // used for tx retrieval. this.blockdb = new bcoin.blockdb(this, { - cache: false + cache: false, + fsync: true }); // Mempool needs access to blockdb. @@ -57,7 +58,8 @@ Fullnode.prototype._init = function _init() { // Chain needs access to blockdb. this.chain = new bcoin.chain(this, { - preload: false + preload: false, + fsync: true }); // Pool needs access to the chain. @@ -135,6 +137,7 @@ Fullnode.prototype._init = function _init() { self.mempool.addBlock(block); }); + if (0) this.chain.on('remove block', function(block) { self.mempool.removeBlock(block); self.walletdb.removeBlock(block); diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index 514f0744..0c747ecb 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -196,7 +196,7 @@ Parser.parseVersion = function parseVersion(p) { // User agent length result = utils.readIntv(p, 80); off = result.off; - agent = p.slice(off, off + result.r); + agent = p.slice(off, off + result.r).toString('ascii'); off += result.r; // Start height @@ -229,7 +229,7 @@ Parser.parseVersion = function parseVersion(p) { local: recv, remote: from, nonce: nonce, - agent: agent.toString('ascii'), + agent: agent, height: height, relay: relay }; @@ -249,7 +249,7 @@ Parser.parseInvList = function parseInvList(p) { for (i = 0, off = 0; i < count; i++, off += 36) { items.push({ type: constants.invByVal[utils.readU32(p, off)], - hash: p.slice(off + 4, off + 36) + hash: new Buffer(p.slice(off + 4, off + 36)) }); } @@ -272,7 +272,7 @@ Parser.parseMerkleBlock = function parseMerkleBlock(p) { hashes = new Array(hashCount); for (i = 0; i < hashCount; i++) - hashes[i] = p.slice(off + i * 32, off + (i + 1) * 32); + hashes[i] = new Buffer(p.slice(off + i * 32, off + (i + 1) * 32)); off = off + 32 * hashCount; flagCount = utils.readIntv(p, off); @@ -282,19 +282,18 @@ Parser.parseMerkleBlock = function parseMerkleBlock(p) { if (off + flagCount > p.length) throw new Error('Invalid flag count'); - flags = p.slice(off, off + flagCount); + flags = new Buffer(p.slice(off, off + flagCount)); return { version: utils.read32(p, 0), - prevBlock: p.slice(4, 36), - merkleRoot: p.slice(36, 68), + prevBlock: new Buffer(p.slice(4, 36)), + merkleRoot: new Buffer(p.slice(36, 68)), ts: utils.readU32(p, 68), bits: utils.readU32(p, 72), nonce: utils.readU32(p, 76), totalTX: utils.readU32(p, 80), hashes: hashes, flags: flags, - _raw: p, _size: p.length }; }; @@ -318,9 +317,9 @@ Parser.parseHeaders = function parseHeaders(p) { start = off; header.version = utils.read32(p, off); off += 4; - header.prevBlock = p.slice(off, off + 32); + header.prevBlock = new Buffer(p.slice(off, off + 32)); off += 32; - header.merkleRoot = p.slice(off, off + 32); + header.merkleRoot = new Buffer(p.slice(off, off + 32)); off += 32; header.ts = utils.readU32(p, off); off += 4; @@ -331,7 +330,6 @@ Parser.parseHeaders = function parseHeaders(p) { r = utils.readIntv(p, off); header.totalTX = r.r; off = r.off; - header._raw = p.slice(start, start + 80); headers.push(header); } @@ -351,7 +349,7 @@ Parser.parseBlock = function parseBlock(p) { totalTX = result.r; for (i = 0; i < totalTX; i++) { - tx = Parser.parseTX(p.slice(off)); + tx = Parser.parseTX(p.slice(off), true); if (!tx) throw new Error('Invalid tx count for block'); tx._offset = off; @@ -362,14 +360,13 @@ Parser.parseBlock = function parseBlock(p) { return { version: utils.read32(p, 0), - prevBlock: p.slice(4, 36), - merkleRoot: p.slice(36, 68), + prevBlock: new Buffer(p.slice(4, 36)), + merkleRoot: new Buffer(p.slice(36, 68)), ts: utils.readU32(p, 68), bits: utils.readU32(p, 72), nonce: utils.readU32(p, 76), totalTX: totalTX, txs: txs, - _raw: p, _size: p.length, _witnessSize: witnessSize }; @@ -412,8 +409,8 @@ Parser.parseBlockCompact = function parseBlockCompact(p) { return { version: version, - prevBlock: p.slice(4, 36), - merkleRoot: p.slice(36, 68), + prevBlock: new Buffer(p.slice(4, 36)), + merkleRoot: new Buffer(p.slice(36, 68)), ts: utils.readU32(p, 68), bits: utils.readU32(p, 72), nonce: utils.readU32(p, 76), @@ -441,10 +438,10 @@ Parser.parseInput = function parseInput(p) { return { _size: off + scriptLen + 4, prevout: { - hash: p.slice(0, 32), + hash: new Buffer(p.slice(0, 32)), index: utils.readU32(p, 32) }, - script: bcoin.script.decode(p.slice(off, off + scriptLen)), + script: bcoin.script.decode(new Buffer(p.slice(off, off + scriptLen))), sequence: utils.readU32(p, off + scriptLen) }; }; @@ -465,7 +462,7 @@ Parser.parseOutput = function parseOutput(p) { return { _size: off + scriptLen, value: utils.read64(p, 0), - script: bcoin.script.decode(p.slice(off, off + scriptLen)) + script: bcoin.script.decode(new Buffer(p.slice(off, off + scriptLen))) }; }; @@ -494,11 +491,11 @@ Parser.parseCoin = function parseCoin(p, extended) { if (off + scriptLen > p.length - (extended ? 37 : 0)) throw new Error('Invalid utxo script length'); - script = bcoin.script.decode(p.slice(off, off + scriptLen)); + script = bcoin.script.decode(new Buffer(p.slice(off, off + scriptLen))); off += scriptLen; if (extended) { - hash = p.slice(off, off + 32); + hash = new Buffer(p.slice(off, off + 32)); off += 32; index = utils.readU32(p, off); @@ -523,17 +520,18 @@ Parser.parseCoin = function parseCoin(p, extended) { }; }; -Parser.parseTX = function parseTX(p) { +Parser.parseTX = function parseTX(p, block) { var off = 0; var inCount, txIn, tx; var outCount, txOut; var version, locktime, i; + var raw; if (p.length < 10) throw new Error('Invalid tx size'); if (Parser.isWitnessTX(p)) - return Parser.parseWitnessTX(p); + return Parser.parseWitnessTX(p, block); version = utils.readU32(p, off); off += 4; @@ -590,13 +588,18 @@ Parser.parseTX = function parseTX(p) { locktime = utils.readU32(p, off); off += 4; + raw = p.length !== off ? p.slice(0, off) : p; + + if (block) + raw = new Buffer(raw); + return { version: version, inputs: txIn, outputs: txOut, locktime: locktime, _witnessSize: 0, - _raw: p.length !== off ? p.slice(0, off) : p, + _raw: raw, _size: off }; }; @@ -608,13 +611,14 @@ Parser.isWitnessTX = function isWitnessTX(p) { return p[4] === 0 && p[5] !== 0; }; -Parser.parseWitnessTX = function parseWitnessTX(p) { +Parser.parseWitnessTX = function parseWitnessTX(p, block) { var off = 0; var inCount, txIn, tx; var outCount, txOut; var marker, flag; var version, locktime, i; var witnessSize = 0; + var raw; if (p.length < 12) throw new Error('Invalid witness tx size'); @@ -699,6 +703,11 @@ Parser.parseWitnessTX = function parseWitnessTX(p) { locktime = utils.readU32(p, off); off += 4; + raw = p.length !== off ? p.slice(0, off) : p; + + if (block) + raw = new Buffer(raw); + return { version: version, marker: marker, @@ -706,7 +715,7 @@ Parser.parseWitnessTX = function parseWitnessTX(p) { inputs: txIn, outputs: txOut, locktime: locktime, - _raw: off !== p.length ? p.slice(0, off) : p, + _raw: raw, _size: off, _witnessSize: witnessSize + 2 }; @@ -725,7 +734,7 @@ Parser.parseWitness = function parseWitness(p) { chunkSize = utils.readIntv(p, off); off = chunkSize.off; chunkSize = chunkSize.r; - item = p.slice(off, off + chunkSize); + item = new Buffer(p.slice(off, off + chunkSize)); off += chunkSize; witness.push(item); if (off > p.length) @@ -768,7 +777,7 @@ Parser.parseReject = function parseReject(p) { off += reasonLen; - data = p.slice(off, off + 32); + data = new Buffer(p.slice(off, off + 32)); return { message: message, @@ -794,7 +803,7 @@ Parser.parseAddress = function parseAddress(p, off, full) { services = utils.readU64(p, off); off += 8; - ip = p.slice(off, off + 16); + ip = new Buffer(p.slice(off, off + 16)); off += 16; port = utils.readU16BE(p, off); diff --git a/lib/bcoin/txdb.js b/lib/bcoin/txdb.js index 71b7dbcb..894d2bc7 100644 --- a/lib/bcoin/txdb.js +++ b/lib/bcoin/txdb.js @@ -720,6 +720,8 @@ TXPool.prototype.unconfirm = function unconfirm(hash, callback) { if (hash.hash) hash = hash.hash('hex'); + callback = utils.ensure(callback); + this.getTX(hash, function(err, tx) { if (err) return callback(err); diff --git a/lib/bcoin/utils.js b/lib/bcoin/utils.js index 818c0c49..c3fca621 100644 --- a/lib/bcoin/utils.js +++ b/lib/bcoin/utils.js @@ -14,6 +14,10 @@ var util = require('util'); * Utils */ +utils.slice = function slice(buf, start, end) { + return new Buffer(buf.slice(start, end)); +}; + utils.toBuffer = function toBuffer(msg, enc) { if (Buffer.isBuffer(msg)) return msg; @@ -1730,3 +1734,28 @@ utils.pad32 = function pad32(num) { assert(num.length === 10); return num; }; + +function SyncBatch(db) { + this.db = db; + this.ops = []; +} + +SyncBatch.prototype.put = function put(key, value) { + this.ops.push({ type: 'put', key: key, value: value, sync: true }); +}; + +SyncBatch.prototype.del = function del(key) { + this.ops.push({ type: 'del', key: key, sync: true }); +}; + +SyncBatch.prototype.write = function write(callback) { + this.db.batch(this.ops, { sync: true }, callback); + this.ops.length = 0; + delete this.ops; +}; + +SyncBatch.prototype.clear = function clear() { + this.ops.length = 0; +}; + +utils.SyncBatch = SyncBatch;