From 7175f81d59aa653b2430f4ceb2bc80d5ab312c9c Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sun, 14 Feb 2016 18:05:21 -0800 Subject: [PATCH] fullnode --- bin/node | 13 + lib/bcoin.js | 2 + lib/bcoin/blockdb.js | 897 +++++++++++++++++++++++++++++++++++ lib/bcoin/coin.js | 4 + lib/bcoin/input.js | 2 + lib/bcoin/mempool.js | 202 ++++++-- lib/bcoin/node.js | 218 +++++++++ lib/bcoin/output.js | 2 + lib/bcoin/protocol/parser.js | 16 +- lib/bcoin/tx.js | 56 ++- 10 files changed, 1353 insertions(+), 59 deletions(-) create mode 100755 bin/node create mode 100644 lib/bcoin/blockdb.js create mode 100644 lib/bcoin/node.js diff --git a/bin/node b/bin/node new file mode 100755 index 00000000..c887d6d8 --- /dev/null +++ b/bin/node @@ -0,0 +1,13 @@ +#!/usr/bin/env node + +var bcoin = require('bcoin'); +var utils = bcoin.utils; +var assert = utils.assert; + +var node = bcoin.node({ + debug: true +}); + +node.on('error', function(err) { + utils.print(err.message); +}); diff --git a/lib/bcoin.js b/lib/bcoin.js index 0f5f2b02..7bd96408 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -53,6 +53,8 @@ bcoin.tx = require('./bcoin/tx'); bcoin.txPool = require('./bcoin/tx-pool'); bcoin.block = require('./bcoin/block'); bcoin.ramdisk = require('./bcoin/ramdisk'); +bcoin.blockdb = require('./bcoin/blockdb'); +bcoin.node = require('./bcoin/node'); bcoin.chainblock = require('./bcoin/chainblock'); bcoin.chaindb = require('./bcoin/chaindb'); bcoin.chain = require('./bcoin/chain'); diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js new file mode 100644 index 00000000..007c75c8 --- /dev/null +++ b/lib/bcoin/blockdb.js @@ -0,0 +1,897 @@ +/** + * db.js - db object for bcoin + * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * https://github.com/indutny/bcoin + */ + +var bcoin = require('../bcoin'); +var utils = bcoin.utils; +var assert = utils.assert; +var levelup = require('levelup'); +var inherits = require('inherits'); +var EventEmitter = require('events').EventEmitter; +var network = bcoin.protocol.network; +var fs = bcoin.fs; + +/** + * BlockDB + */ + +function BlockDB(options) { + var self = this; + + if (!(this instanceof BlockDB)) + return new BlockDB(options); + + EventEmitter.call(this); + + if (!options) + options = {}; + + if (typeof options === 'string') + options = { file: options }; + + if (!options.file) + options.file = process.env.HOME + '/bcoin-server-' + network.type + '.ldb'; + + this.options = options; + this.data = new BlockData(); + this.index = levelup(options.file, { + keyEncoding: 'ascii', + valueEncoding: 'binary', + cacheSize: 16 * 1024 * 1024 + }); +} + +inherits(BlockDB, EventEmitter); + +BlockDB.prototype.get = function get(key, options, callback) { + return this.index.get(key, options, callback); +}; + +BlockDB.prototype.put = function put(key, value, options, callback) { + return this.index.put(key, value, options, callback); +}; + +BlockDB.prototype.del = function del(key, options, callback) { + return this.index.del(key, options, callback); +}; + +BlockDB.prototype.batch = function batch() { + return this.index.batch(); +}; + +BlockDB.prototype.createOffset = function createOffset(size, offset, height) { + var buf = new Buffer(16); + utils.writeU32(buf, size, 0); + utils.writeU64(buf, offset, 4); + utils.writeU32(buf, height, 12); + return buf; +}; + +BlockDB.prototype.parseOffset = function parseOffset(data, height) { + return { + size: utils.readU32(data, 0), + offset: utils.readU64(data, 4).toNumber(), + height: utils.readU32(data, 12) + }; +}; + +BlockDB.prototype.saveBlock = function saveBlock(block, callback) { + var self = this; + var batch = this.index.batch(); + + this.data.saveAsync(block._raw, function(err, data) { + if (err) + return callback(err); + + var blockOffset = self.createOffset(data.size, data.offset, block.height); + + var batch = self.index.batch(); + batch.put('b/b/' + block.hash('hex'), blockOffset); + batch.put('b/h/' + block.height, blockOffset); + + block.txs.forEach(function(tx, i) { + var txOffset = self.createOffset(tx._size, data.offset + tx._offset, block.height); + var hash = tx.hash('hex'); + var uniq = {}; + + batch.put('t/t/' + hash, txOffset); + + tx.inputs.forEach(function(input) { + var type = input.getType(); + var address = input.getAddress(); + var uaddr; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + if (uaddr) + batch.put('t/a/' + uaddr + '/' + hash, txOffset); + + if (address) + batch.del('u/a/' + address + '/' + input.prevout.hash + '/' + input.prevout.index); + + batch.del('u/t/' + input.prevout.hash + '/' + input.prevout.index); + }); + + tx.outputs.forEach(function(output) { + var type = output.getType(); + var address = output.getAddress(); + var uaddr; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + var coinOffset = self.createOffset( + output._size, + data.offset + tx._offset + output._offset, + block.height); + // console.log(self.parseOffset(coinOffset)); + + if (uaddr) + batch.put('t/a/' + uaddr + '/' + hash, txOffset); + + if (address) + batch.put('u/a/' + address + '/' + hash + '/' + i, coinOffset); + + batch.put('u/t/' + hash + '/' + i, coinOffset); + }); + }); + + batch.write(callback); + }); +}; + +BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { + var self = this; + + this.getBlock(hash, function(err, block) { + var batch, pending; + + if (err) + return callback(err); + + if (!block) + return callback(); + + pending = block.txs.length; + + batch = self.index.batch(); + + batch.del('b/b/' + hash); + batch.del('b/h/' + block.height); + + function done() { + batch.write(function(err) { + if (err) + return callback(err); + return callback(null, block); + }); + } + + if (!pending) + return done(); + + block.txs.forEach(function(tx, i) { + var hash = tx.hash('hex'); + var uniq = {}; + + batch.del('t/t/' + hash); + + self.fillTX2(tx, function(err) { + if (err) + return callback(err); + + tx.inputs.forEach(function(input) { + var type = input.getType(); + var address = input.getAddress(); + var uaddr; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + if (uaddr) + batch.del('t/a/' + uaddr + '/' + hash); + + var coinOffset = self.createOffset( + input.output._size, + block._fileOffset + tx._offset + input.output._offset, + block.height); + + if (address) + batch.put('u/a/' + address + '/' + input.output.hash + '/' + input.output.index, coinOffset); + + batch.put('u/t/' + input.output.hash + '/' + input.output.index, coinOffset); + }); + + tx.outputs.forEach(function(output) { + var type = output.getType(); + var address = output.getAddress(); + var uaddr; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + if (uaddr) + batch.del('t/a/' + uaddr + '/' + hash); + + if (address) + batch.del('u/a/' + address + '/' + hash + '/' + i); + + batch.del('u/t/' + hash + '/' + i); + }); + + if (!--pending) + done(); + }); + }); + }); +}; + +BlockDB.prototype.fillTX = function fillTX(tx, callback) { + var self = this; + var pending = tx.inputs.length; + + tx.inputs.forEach(function(input) { + if (input.output) { + if (!--pending) + callback(null, tx); + return; + } + self.getCoin(input.prevout.hash, input.prevout.index, function(err, coin) { + if (err) + return callback(err); + + if (coin) + input.output = coin; + + if (!--pending) + callback(null, tx); + }); + }); +}; + +BlockDB.prototype.fillTX2 = function fillTX2(tx, callback) { + var self = this; + var pending = tx.inputs.length; + + tx.inputs.forEach(function(input) { + if (input.output) { + if (!--pending) + callback(null, tx); + return; + } + self.getTX(input.prevout.hash, function(err, tx) { + if (err) + return callback(err); + + if (tx) + input.output = bcoin.coin(tx, input.prevout.index); + + if (!--pending) + callback(null, tx); + }); + }); +}; + +BlockDB.prototype.getCoinsByAddress = function getCoinsByAddress(addresses, callback) { + var self = this; + var coins = []; + var pending; + + if (typeof addresses === 'string') + addresses = [addresses]; + + addresses = utils.uniqs(addresses); + + var pending = addresses.length; + + addresses.forEach(function(address) { + self._getCoinsByAddress(address, function(err, coin) { + if (err) + return callback(err); + + if (coin) + coins = coins.concat(coin); + + if (!--pending) + return callback(null, coins); + }); + }); +}; + +BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, callback) { + var self = this; + var pending = 0; + var coins = []; + var done = false; + + callback = utils.asyncify(callback); + + var stream = this.index.createReadStream({ + start: 'u/a/' + address, + end: 'u/a/' + address + '~' + }); + + stream.on('data', function(data) { + var parts = data.key.split('/').slice(3); + var hash = parts[0]; + var index = +parts[1]; + var record = self.parseOffset(data.value); + pending++; + self.data.getAsync(record.size, record.offset, function(err, data) { + if (err) + return callback(err); + + if (data) { + var coin = bcoin.coin.fromRaw(data, true); + coin.hash = hash; + coin.index = index; + coin.height = record.height; + coins.push(coin); + } + + pending--; + + if (done) { + if (!pending) + return callback(null, coins); + } + }); + }); + + stream.on('error', function(err) { + return callback(err); + }); + + stream.on('end', function() { + done = true; + if (!pending) + return callback(null, coins); + }); +}; + +BlockDB.prototype.getCoin = function getCoin(hash, index, callback) { + var self = this; + var id = 'u/t/' + hash + '/' + index; + + this.index.get(id, { valueEncoding: 'binary' }, function(err, record) { + if (err) { + if (err.type === 'NotFoundError') + return callback(); + return callback(err); + } + + record = self.parseOffset(record); + + self.data.getAsync(record.size, record.offset, function(err, data) { + var coin; + + if (err) + return callback(err); + + if (data) { + coin = bcoin.coin.fromRaw(data, true); + coin.hash = hash; + coin.index = index; + coin.height = record.height; + } + + return callback(null, coin); + }); + }); +}; + +BlockDB.prototype.getTXByAddress = function getTXByAddress(addresses, callback) { + var self = this; + var txs = []; + var pending; + + if (typeof addresses === 'string') + addresses = [addresses]; + + addresses = utils.uniqs(addresses); + + pending = addresses.length; + + if (!pending) + return callback(null, txs); + + addresses.forEach(function(address) { + self._getTXByAddress(address, function(err, tx) { + if (err) + return callback(err); + + if (tx) + txs = txs.concat(tx); + + if (!--pending) + return callback(null, txs); + }); + }); +}; + +BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback) { + var self = this; + var pending = 0; + var txs = []; + var done = false; + var stream; + + callback = utils.asyncify(callback); + + stream = this.index.createReadStream({ + start: 't/a/' + address, + end: 't/a/' + address + '~' + }); + + stream.on('data', function(data) { + var parts = data.key.split('/').slice(3); + var hash = parts[0]; + var record = self.parseOffset(data.value); + + pending++; + + self.data.getAsync(record.size, record.offset, function(err, data) { + var tx, entry; + + if (err) + return callback(err); + + if (data) { + tx = bcoin.tx.fromRaw(data); + entry = bcoin.chain.global.db.get(record.height); + tx.height = record.height; + tx.ts = entry.ts; + tx.block = entry.hash; + txs.push(tx); + } + + pending--; + + if (done) { + if (!pending) + return callback(null, txs); + } + }); + }); + + stream.on('error', function(err) { + return callback(err); + }); + + stream.on('end', function() { + done = true; + if (!pending) + return callback(null, txs); + }); +}; + +BlockDB.prototype.getTX = function getTX(hash, callback) { + var self = this; + var id = 't/t/' + hash; + + this.index.get(id, { valueEncoding: 'binary' }, function(err, record) { + if (err) { + if (err.type === 'NotFoundError') + return callback(); + return callback(err); + } + + record = self.parseOffset(record); + + self.data.getAsync(record.size, record.offset, function(err, data) { + var tx, entry; + + if (err) + return callback(err); + + if (data) { + tx = bcoin.tx.fromRaw(data); + entry = bcoin.chain.global.db.get(record.height); + tx.height = record.height; + tx.ts = entry.ts; + tx.block = entry.hash; + } + + return callback(null, tx); + }); + }); +}; + +BlockDB.prototype.getBlock = function getBlock(hash, callback) { + var self = this; + var id = 'b/b/' + value; + + if (typeof hash === 'number') + id = 'b/h/' + value; + + this.index.get(id, { valueEncoding: 'binary' }, function(err, record) { + if (err) { + if (err.type === 'NotFoundError') + return callback(); + return callback(err); + } + + record = self.parseOffset(record); + + self.data.getAsync(record.size, record.offset, function(err, data) { + var block; + + if (err) + return callback(err); + + if (data) { + block = bcoin.block.fromRaw(data); + block._fileOffset = record.offset; + block.height = record.height; + } + + return callback(null, block); + }); + }); +}; + +/** + * BlockData + */ + +function BlockData(chain, options) { + if (!(this instanceof BlockData)) + return new BlockData(chain, options); + + if (!options) + options = {}; + + this.options = options; + this.chain = chain; + this.file = options.file; + + if (!this.file) + this.file = process.env.HOME + '/bcoin-' + network.type + '.data'; + + this._queue = []; + this._cache = {}; + this._bufferPool = { used: {} }; + this.size = 0; + this.fd = null; + + this._init(); +} + +BlockData.prototype._init = function _init() { + if (!bcoin.fs) { + utils.debug('`fs` module not available. Falling back to ramdisk.'); + this.ramdisk = bcoin.ramdisk(new Buffer([]), 40 * 1024 * 1024); + return; + } + + if (+process.env.BCOIN_FRESH === 1) { + try { + fs.unlinkSync(this.file); + } catch (e) { + ; + } + } + + if (!this.exists()) { + fs.writeFileSync(this.file, new Buffer(0)); + fs.truncateSync(this.file, 0); + } + + this.size = this.getSize(); + + this.fd = fs.openSync(this.file, 'r+'); +}; + +BlockData.prototype._malloc = function(size) { + if (!this._bufferPool[size]) + this._bufferPool[size] = new Buffer(size); + + if (this._bufferPool.used[size] === this._bufferPool[size]) + return new Buffer(size); + + this._bufferPool.used[size] = this._bufferPool[size]; + + return this._bufferPool[size]; +}; + +BlockData.prototype._free = function(buf) { + if (this._bufferPool.used[buf.length] === buf) { + assert(this._bufferPool[buf.length] === buf); + delete this._bufferPool.used[buf.length]; + } +}; + +BlockData.prototype.exists = function exists() { + if (!bcoin.fs) + return true; + + try { + fs.statSync(this.file); + return true; + } catch (e) { + return false; + } +}; + +BlockData.prototype.getSize = function getSize() { + if (!bcoin.fs) + return this.ramdisk.size; + + try { + return fs.statSync(this.file).size; + } catch (e) { + return 0; + } +}; + +BlockData.prototype.getSync = function getSync(size, offset) { + var hash = size + '/' + offset; + + if (this._cache[hash]) + return this._cache[hash]; + + if (this._queue[hash]) + return this._queue[hash]; + + return this._readSync(size, offset); +}; + +BlockData.prototype.getAsync = function getAsync(size, offset, callback) { + var self = this; + var hash = size + '/' + offset; + + callback = utils.asyncify(callback); + + if (this._cache[hash]) + return callback(null, this._cache[hash]); + + if (this._queue[hash]) + return callback(null, this._queue[hash]); + + return this._readAsync(size, offset, callback); +}; + +BlockData.prototype.saveSync = function saveSync(data) { + var self = this; + var offset = this.size; + var hash = data + '/' + offset; + + this._writeSync(data, offset); + + return { size: data.length, offset: offset }; +}; + +BlockData.prototype.saveAsync = function saveAsync(data, callback) { + var self = this; + var offset = this.size; + var hash = data + '/' + offset; + + callback = utils.asyncify(callback); + + // Something is already writing. Cancel it + // and synchronously write the data after + // it cancels. + if (this._queue[hash]) { + this._queue[hash] = data; + return callback(null, { size: data.length, offset: offset }); + } + + // Speed up writes by doing them asynchronously + // and keeping the data to be written in memory. + this._queue[hash] = data; + + return this._writeAsync(data, offset, function(err, success) { + if (err) + return callback(err); + + var item = self._queue[hash]; + + // Something tried to write here but couldn't. + // Synchronously write it and get it over with. + try { + if (item && item !== data) + success = self._writeSync(item, offset); + } catch (e) { + err = e; + } + + delete self._queue[hash]; + + return callback(null, { size: data.length, offset: offset }); + }); +}; + +BlockData.prototype.truncate = function truncate(height) { + this.size = (height + 1) * BLOCK_SIZE; + this.tip = height; + + if (!bcoin.fs) { + this.ramdisk.truncate(this.size); + return; + } + + fs.ftruncateSync(this.fd, this.size); +}; + +BlockData.prototype._readSync = function _readSync(size, offset) { + var index = 0; + var data, bytes; + + if (offset < 0 || offset == null) + return; + + if (!bcoin.fs) + return this.ramdisk.read(size, offset); + + data = this._malloc(size); + + try { + while (bytes = fs.readSync(this.fd, data, index, size, offset)) { + index += bytes; + size -= bytes; + offset += bytes; + if (index === data.length) { + this._free(data); + return data; + } + } + } catch (e) { + this._free(data); + throw e; + } + + this._free(data); + + throw new Error('_readSync() failed.'); +}; + +BlockData.prototype._readAsync = function _readAsync(size, offset, callback) { + var self = this; + var index = 0; + var data, bytes; + + callback = utils.asyncify(callback); + + if (offset < 0 || offset == null) + return callback(); + + if (!bcoin.fs) + return callback(null, this.ramdisk.read(size, offset)); + + data = this._malloc(size); + + (function next() { + fs.read(self.fd, data, index, size, offset, function(err, bytes) { + if (err) { + self._free(data); + return callback(err); + } + + index += bytes; + size -= bytes; + offset += bytes; + + if (index === data.length) { + self._free(data); + return callback(null, data); + } + + next(); + }); + })(); +}; + +BlockData.prototype._writeSync = function _writeSync(data, offset) { + var size = data.length; + var added = Math.max(0, (offset + data.length) - this.size); + var index = 0; + var bytes; + + if (offset < 0 || offset == null) + return false; + + if (!bcoin.fs) { + this.size += added; + this.ramdisk.write(data, offset); + return; + } + + try { + while (bytes = fs.writeSync(this.fd, data, index, size, offset)) { + index += bytes; + size -= bytes; + offset += bytes; + if (index === data.length) { + this.size += added; + return true; + } + } + } catch (e) { + throw e; + } + + throw new Error('_writeSync() failed.'); +}; + +BlockData.prototype._writeAsync = function _writeAsync(data, offset, callback) { + var self = this; + var added = Math.max(0, (offset + data.length) - this.size); + var size = data.length; + var index = 0; + + callback = utils.asyncify(callback); + + if (offset < 0 || offset == null) + return callback(null, false); + + if (!bcoin.fs) { + this.size += added; + this.ramdisk.write(data, offset); + return callback(null, true); + } + + this.size += added; + + (function next() { + fs.write(self.fd, data, index, size, offset, function(err, bytes) { + if (err) { + self.size -= (added - index); + return callback(err); + } + + index += bytes; + size -= bytes; + offset += bytes; + + if (index === data.length) + return callback(null, true); + + next(); + }); + })(); +}; + +/** + * Expose + */ + +module.exports = BlockDB; diff --git a/lib/bcoin/coin.js b/lib/bcoin/coin.js index 51439cfe..e2102e0c 100644 --- a/lib/bcoin/coin.js +++ b/lib/bcoin/coin.js @@ -34,6 +34,8 @@ function Coin(tx, index) { this.height = tx.height; this.value = tx.outputs[index].value; this.script = tx.outputs[index].script; + this._offset = tx.outputs[index]._offset; + this._size = tx.outputs[index]._size; this.hash = tx.hash('hex'); this.index = index; this.spent = false; @@ -46,6 +48,8 @@ function Coin(tx, index) { this.hash = options.hash; this.index = options.index; this.spent = options.spent; + this._size = options._size || 0; + this._offset = options._offset || 0; } if (utils.isBuffer(this.hash)) diff --git a/lib/bcoin/input.js b/lib/bcoin/input.js index e136ecae..5c391530 100644 --- a/lib/bcoin/input.js +++ b/lib/bcoin/input.js @@ -39,6 +39,8 @@ function Input(options) { this.script = options.script ? options.script.slice() : []; this.sequence = options.sequence == null ? 0xffffffff : options.sequence; + this._size = options._size || 0; + this._offset = options._offset || 0; // Legacy if (options.seq != null) diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index 8769fcc7..aa1a3109 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -19,7 +19,7 @@ var fs = bcoin.fs; * Mempool */ -function Mempool(pool, options) { +function Mempool(node, options) { if (!(this instanceof Mempool)) return new Mempool(pool, options); @@ -27,11 +27,13 @@ function Mempool(pool, options) { options = {}; this.options = options; - this.pool = pool; - this.storage = bcoin.db; + this.node = node; + this.pool = node.pool; + this.storage = node.storage; this.txs = {}; - this.prevout = {}; + this.spent = {}; + this.addresses = {}; this.size = 0; this.count = 0; this.locked = false; @@ -39,42 +41,119 @@ function Mempool(pool, options) { this._init(); } -Mempool.prototype._init = function _init() { - var self = this; +inherits(Mempool, EventEmitter); +Mempool.prototype._init = function _init() { + ; +}; + +Mempool.prototype.addBlock = function addBlock(block) { + var self = this; // Remove now-mined transactions - this.pool.on('block', function(block) { - block.txs.forEach(function(tx) { - var mtx = self.get(tx); + block.txs.forEach(function(tx) { + var mtx = self.get(tx); + if (!mtx) + return; + + mtx.ps = 0; + mtx.ts = block.ts; + mtx.block = block.hash('hex'); + mtx.network = true; + + self.removeTX(mtx); + }); +}; + +Mempool.prototype.removeBlock = function removeBlock(block) { + var self = this; + block.txs.forEach(function(tx) { + var hash = tx.hash('hex'); + // Remove anything that tries to redeem these outputs + tx.outputs.forEach(function(output, i) { + var mtx = self.spent[hash + '/' + i]; if (!mtx) return; - mtx.ps = 0; - mtx.ts = block.ts; - mtx.block = block.hash('hex'); - - self.remove(mtx); + self.removeTX(mtx); }); }); }; -Mempool.prototype.get = function get(hash) { +Mempool.prototype.get = +Mempool.prototype.getTX = function getTX(hash) { if (hash instanceof bcoin.tx) hash = hash.hash('hex'); return this.txs[hash]; }; +Mempool.prototype.getCoin = function getCoin(hash, index) { + var tx = this.get(hash); + if (!tx) + return; + + return bcoin.coin(tx, index); +}; + +Mempool.prototype.isSpent = function isSpent(hash, index) { + return !!this.spent[hash + '/' + index]; +}; + +Mempool.prototype.getCoinsByAddress = function getCoinsByAddress(addresses) { + var txs = this.getByAddress(addresses); + return txs.reduce(function(out, tx) { + return out.concat(tx.outputs.map(function(output, i) { + return bcoin.coin(tx, i); + })); + }, []); +}; + +Mempool.prototype.getByAddress = +Mempool.prototype.getTXByAddress = function getTXByAddress(addresses) { + var self = this; + var txs = []; + var uniq = {}; + + if (typeof addresses === 'string') + addresses = [addresses]; + + addresses = utils.uniqs(addresses); + + addresses.forEach(function(address) { + var map = self.addresses[address]; + if (!map) + return; + + Object.keys(map).forEach(function(hash) { + var tx; + + if (uniq[hash]) + return; + + uniq[hash] = true; + + tx = self.get(hash); + assert(tx); + + txs.push(tx); + }); + }); + + return txs; +}; + Mempool.prototype.getAll = function getAll(hash) { return Object.keys(this.txs).map(function(key) { return this.txs[key]; }, this); }; -Mempool.prototype.has = function has(hash) { +Mempool.prototype.has = +Mempool.prototype.hasTX = function hasTX(hash) { return !!this.get(hash); }; -Mempool.prototype.add = function add(tx, peer, callback) { +Mempool.prototype.add = +Mempool.prototype.addTX = function addTX(tx, peer, callback) { var self = this; var hash = tx.hash('hex'); @@ -105,6 +184,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { return callback(err); if (!tx.hasPrevout()) { + return callback(new Error('Previous outputs not found.')); peer.reject({ data: tx.hash(), reason: 'no-prevout' @@ -114,6 +194,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { } if (!tx.isStandard()) { + return callback(new Error('TX is not standard.')); peer.reject({ data: tx.hash(), reason: 'non-standard' @@ -123,6 +204,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { } if (!tx.isStandardInputs()) { + return callback(new Error('TX inputs are not standard.')); peer.reject({ data: tx.hash(), reason: 'non-standard-inputs' @@ -132,6 +214,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { } if (tx.getOutputValue().cmp(tx.getInputValue()) > 0) { + return callback(new Error('TX is spending coins that it does not have.')); peer.reject({ data: tx.hash(), reason: 'nonexistent-coins' @@ -143,6 +226,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { height = self.pool.chain.height() + 1; ts = utils.now(); if (!tx.isFinal(height, ts)) { + return callback(new Error('TX is not final.')); peer.reject({ data: tx.hash(), reason: 'not-final' @@ -154,6 +238,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; if (input.output.spent) { + return callback(new Error('TX is spending old outputs.')); peer.reject({ data: tx.hash(), reason: 'old-outputs' @@ -161,7 +246,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { pool.setMisbehavior(peer, 100); return callback(new Error('TX is spending old outputs.')); } - dup = self.prevout[input.prevout.hash]; + dup = self.spent[input.prevout.hash + '/' + input.prevout.index]; if (dup) { // Replace-by-fee if (input.sequence === 0xffffffff - 1) { @@ -170,6 +255,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { continue; } } + return callback(new Error('TX is double spending.')); peer.reject({ data: tx.hash(), reason: 'double-spend' @@ -182,6 +268,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; if (output.value.cmpn(0) < 0) { + return callback(new Error('TX is spending negative coins.')); peer.reject({ data: tx.hash(), reason: 'negative-value' @@ -192,6 +279,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { } if (!tx.verify(true)) { + return callback(new Error('TX did not verify.')); peer.reject({ data: tx.hash(), reason: 'script-failed' @@ -202,7 +290,7 @@ Mempool.prototype.add = function add(tx, peer, callback) { for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - self.prevout[input.prevout.hash] = tx; + self.spent[input.prevout.hash + '/' + input.prevout.index] = tx; } // Possibly do something bitcoinxt-like here with priority @@ -212,11 +300,28 @@ Mempool.prototype.add = function add(tx, peer, callback) { self.count++; self.size += tx.getSize(); - self.storage.saveMempoolTX(tx, function(err) { - if (err) - return callback(err); + tx.inputs.forEach(function(input) { + var address = input.getAddress(); - return callback(); + if (!address) + return; + + if (!self.addresses[address]) + self.addresses[address] = {}; + + self.addresses[address][hash] = true; + }); + + tx.outputs.forEach(function(output) { + var address = output.getAddress(); + + if (!address) + return; + + if (!self.addresses[address]) + self.addresses[address] = {}; + + self.addresses[address][hash] = true; }); }); }; @@ -224,35 +329,38 @@ Mempool.prototype.add = function add(tx, peer, callback) { // Lock a tx to prevent race conditions Mempool.prototype._lockTX = function _lockTX(tx) { var hash = tx.hash('hex'); - var i, input; + var i, input, id; if (!this.txs[hash]) this.txs[hash] = tx; for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - if (!this.prevout[input.prevout.hash]) - this.prevout[input.prevout.hash] = tx; + id = input.prevout.hash + '/' + input.prevout.index; + if (!this.spent[id]) + this.spent[id] = tx; } }; Mempool.prototype._unlockTX = function _unlockTX(tx) { var hash = tx.hash('hex'); - var i, input; + var i, input, id; if (this.txs[hash] === tx) delete this.txs[hash]; for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - if (this.prevout[input.prevout.hash] === tx) - delete this.prevout[input.prevout.hash]; + id = input.prevout.hash + '/' + input.prevout.index; + if (this.spent[id] === tx) + delete this.spent[id]; } }; -Mempool.prototype.remove = function remove(hash, callback) { +Mempool.prototype.remove = +Mempool.prototype.removeTX = function removeTX(hash, callback) { var self = this; - var tx, input; + var tx, input, id; callback = utils.asyncify(callback); @@ -266,8 +374,9 @@ Mempool.prototype.remove = function remove(hash, callback) { for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - if (this.prevout[input.prevout.hash] === tx) - delete this.prevout[input.prevout.hash]; + id = input.prevout.hash + '/' + input.prevout.index; + if (this.spent[id] === tx) + delete this.spent[id]; } delete this.txs[hash]; @@ -275,11 +384,30 @@ Mempool.prototype.remove = function remove(hash, callback) { this.count--; this.size -= tx.getSize(); - this.storage.removeMempoolTX(tx, function(err) { - if (err) - return callback(err); + tx.inputs.forEach(function(input) { + var address = input.getAddress(); - return callback(); + if (!address) + return; + + if (self.addresses[address]) { + delete self.addresses[address][hash]; + if (Object.keys(self.addresses[address]).length === 0) + delete self.addresses[address]; + } + }); + + tx.outputs.forEach(function(output) { + var address = output.getAddress(); + + if (!address) + return; + + if (self.addresses[address]) { + delete self.addresses[address][hash]; + if (Object.keys(self.addresses[address]).length === 0) + delete self.addresses[address]; + } }); }; diff --git a/lib/bcoin/node.js b/lib/bcoin/node.js new file mode 100644 index 00000000..c0c414c3 --- /dev/null +++ b/lib/bcoin/node.js @@ -0,0 +1,218 @@ +/** + * node.js - full node for bcoin + * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * https://github.com/indutny/bcoin + */ + +var inherits = require('inherits'); +var EventEmitter = require('events').EventEmitter; +var bcoin = require('../bcoin'); +var bn = require('bn.js'); +var constants = bcoin.protocol.constants; +var network = bcoin.protocol.network; +var utils = bcoin.utils; +var assert = utils.assert; +var fs = bcoin.fs; + +/** + * Node + */ + +function Node(options) { + if (!(this instanceof Node)) + return new Node(options); + + EventEmitter.call(this); + + if (!options) + options = {}; + + this.options = options; + + if (this.options.debug) + bcoin.debug = this.options.debug; + + if (this.options.network) + network.set(this.options.network); + + this.storage = null; + this.mempool = null; + this.pool = null; + this.chain = null; + + Node.global = this; + + this._init(); +} + +inherits(Node, EventEmitter); + +Node.prototype._init = function _init() { + var self = this; + + if (!this.options.pool) + this.options.pool = {}; + + this.options.pool.type = 'full'; + + this.storage = new bcoin.blockdb(this.options.storage); + this.mempool = new bcoin.mempool(this, this.options.mempool); + this.pool = new bcoin.pool(this.options.pool); + this.chain = this.pool.chain; + + this.pool.on('block', function(block, peer) { + self.storage.saveBlock(block, function(err) { + if (err) + throw err; + + self.mempool.addBlock(block); + var hash = block.txs[0].hash('hex'); + if (0) + self.storage.getTX(hash, function(err, tx) { + if (err) throw err; + utils.print(tx); + }); + self.storage.getCoin(hash, 0, function(err, tx) { + if (err) throw err; + utils.print(tx); + }); + }); + }); + + this.mempool.on('error', function(err) { + self.emit('error', err); + }); + + this.chain.on('error', function(err) { + self.emit('error', err); + }); + + this.pool.on('error', function(err) { + self.emit('error', err); + }); + + this.pool.on('fork', function(a, b) { + [a, b].forEach(function(hash) { + self.storage.removeBlock(hash, function(err, block) { + if (err) + throw err; + + if (!block) + return; + + self.mempool.removeBlock(block); + }); + }); + }); + + this.pool.on('tx', function(tx, peer) { + assert(tx.ts === 0); + self.mempool.addTX(tx, peer); + }); + + this.pool.startSync(); +}; + +Node.prototype.getCoin = function getCoin(hash, index, callback) { + var self = this; + var coin; + + callback = utils.asyncify(callback); + + coin = this.mempool.getCoin(hash, index); + if (coin) + return callback(null, coin); + + if (this.mempool.isSpent(hash, index)) + return callback(null, null); + + this.storage.getCoin(hash, index, function(err, coin) { + if (err) + return callback(err); + + if (!coin) + return; + + return callback(null, coin); + }); +}; + +Node.prototype.getCoinByAddress = function getCoinsByAddress(addresses, callback) { + var self = this; + var mempool; + + callback = utils.asyncify(callback); + + mempool = this.mempool.getCoinsByAddress(addresses); + + this.storage.getCoinsByAddress(addresses, function(err, coins) { + if (err) + return callback(err); + + return callback(null, mempool.concat(coins.filter(function(coin) { + if (self.mempool.isSpent(coin.hash, coin.index)) + return false; + return true; + }))); + }); +}; + +Node.prototype.getTX = function getTX(hash, callback) { + var self = this; + var tx; + + callback = utils.asyncify(callback); + + tx = this.mempool.getTX(hash); + if (tx) + return callback(null, tx); + + this.storage.getTX(hash, function(err, tx) { + if (err) + return callback(err); + + return callback(null, tx); + }); +}; + +Node.prototype.isSpent = function isSpent(hash, index, callback) { + var self = this; + + callback = utils.asyncify(callback); + + if (this.mempool.isSpent(hash, index)) + return callback(null, true); + + this.storage.getCoin(hash, index, function(err, coin) { + if (err) + return callback(err); + + return callback(null, coin ? false : true); + }); +}; + +Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) { + var self = this; + var mempool; + + callback = utils.asyncify(callback); + + mempool = this.mempool.getTXByAddress(addresses); + + this.storage.getTXByAddress(addresses, function(err, txs) { + if (err) + return callback(err); + + return callback(null, mempool.concat(txs)); + }); +}; + +Node.prototype.fillTX = function fillTX(tx, callback) { + this.storage.fillTX(tx, callback); +}; + +/** + * Expose + */ + +module.exports = Node; diff --git a/lib/bcoin/output.js b/lib/bcoin/output.js index 56695e81..f8cb2fae 100644 --- a/lib/bcoin/output.js +++ b/lib/bcoin/output.js @@ -28,6 +28,8 @@ function Output(options) { this.value = utils.satoshi(value || new bn(0)); this.script = options.script ? options.script.slice() : []; + this._size = options._size || 0; + this._offset = options._offset || 0; // For safety: do not allow usage of // Numbers, do not allow negative values. diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index 9bfc8080..34e5d2b6 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -340,7 +340,8 @@ Parser.prototype.parseBlock = function parseBlock(p) { tx = this.parseTX(p.slice(off)); if (!tx) return this._error('Invalid tx count for block'); - off += tx._off; + tx._offset = off; + off += tx._size; txs.push(tx); } @@ -372,7 +373,7 @@ Parser.prototype.parseTXIn = function parseTXIn(p) { return this._error('Invalid tx_in script length'); return { - size: off + scriptLen + 4, + _size: off + scriptLen + 4, prevout: { hash: utils.toArray(p.slice(0, 32)), index: utils.readU32(p, 32) @@ -396,7 +397,7 @@ Parser.prototype.parseTXOut = function parseTXOut(p) { return this._error('Invalid tx_out script length'); return { - size: off + scriptLen, + _size: off + scriptLen, value: utils.read64(p, 0), script: bcoin.script.decode(utils.toArray(p.slice(off, off + scriptLen))) }; @@ -428,7 +429,8 @@ Parser.prototype.parseTX = function parseTX(p) { return; txIn[i] = tx; - off += tx.size; + tx._offset = off; + off += tx._size; if (off + 5 > p.length) return this._error('Invalid tx_in offset'); @@ -450,7 +452,8 @@ Parser.prototype.parseTX = function parseTX(p) { return; txOut[i] = tx; - off += tx.size; + tx._offset = off; + off += tx._size; if (off + 4 > p.length) return this._error('Invalid tx_out offset'); @@ -462,8 +465,7 @@ Parser.prototype.parseTX = function parseTX(p) { inputs: txIn, outputs: txOut, locktime: utils.readU32(p, off), - _off: off + 4, - _size: p.length + _size: off + 4 }; }; diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index b2a1beee..4c3366a9 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -37,6 +37,7 @@ function TX(data, block) { this._raw = data._raw || null; this._size = data._size || 0; + this._offset = data._offset || 0; this.height = data.height != null ? data.height : -1; this.network = data.network || false; @@ -101,14 +102,30 @@ TX.prototype.clone = function clone() { return new TX(this); }; +TX.prototype.isStatic = function isStatic() { + return this.ts !== 0 || this.network; +}; + TX.prototype.hash = function hash(enc, force) { - var h = utils.dsha256(this.render(force)); - return enc === 'hex' ? utils.toHex(h) : h; + var hash; + + if (!force && this._hash) + return enc === 'hex' ? utils.toHex(this._hash) : this._hash; + + if (!force && this.isStatic() && this._raw) + hash = utils.dsha256(this._raw); + else + hash = utils.dsha256(this.render(true)); + + if (this.isStatic()) + this._hash = hash; + + return enc === 'hex' ? utils.toHex(hash) : hash; }; TX.prototype.render = function render(force) { - if (!force && this.network && this._raw) - return utils.toArray(this._raw).slice(); + if (!force && this.isStatic() && this._raw) + return utils.toArray(this._raw); return bcoin.protocol.framer.tx(this); }; @@ -145,6 +162,7 @@ TX.prototype._addInput = function _addInput(options, index) { if (options.seq != null) options.sequence = options.seq; + var isInput; if (!options.prevout) { if (options instanceof bcoin.coin) { options = { @@ -167,6 +185,8 @@ TX.prototype._addInput = function _addInput(options, index) { sequence: options.sequence }; } + } else { + isInput = true; } input = bcoin.input({ @@ -177,7 +197,9 @@ TX.prototype._addInput = function _addInput(options, index) { }, output: options.output, script: options.script, - sequence: options.sequence + sequence: options.sequence, + _size: isInput ? options._size : null, + _offset: isInput ? options._offset : null }); // Try modifying existing input first @@ -673,7 +695,9 @@ TX.prototype.addOutput = function addOutput(obj, value) { output = bcoin.output({ tx: this, value: options.value, - script: options.script + script: options.script, + _size: options._size, + _offset: options._offset }); this.outputs.push(output); @@ -1762,7 +1786,7 @@ TX.prototype.inspect = function inspect() { return copy; }; -TX.prototype.toJSON = function toJSON() { +TX.prototype.toJSON = function toJSON(coins) { // Compact representation return { v: 1, @@ -1774,9 +1798,9 @@ TX.prototype.toJSON = function toJSON() { network: this.network, relayedBy: this.relayedBy, changeIndex: this.changeIndex, - coins: this.inputs.map(function(input) { + coins: coins ? this.inputs.map(function(input) { return input.output ? input.output.toJSON() : null; - }), + }) : null, tx: utils.toHex(this.render()) }; }; @@ -1804,12 +1828,14 @@ TX.fromJSON = function fromJSON(json) { tx.block = json.block || null; tx.ps = json.ps; - json.coins.forEach(function(output, i) { - if (!output) - return; + if (json.coins) { + json.coins.forEach(function(output, i) { + if (!output) + return; - tx.inputs[i].output = bcoin.coin.fromJSON(output); - }); + tx.inputs[i].output = bcoin.coin.fromJSON(output); + }); + } return tx; }; @@ -1817,7 +1843,7 @@ TX.fromJSON = function fromJSON(json) { TX.prototype.toRaw = function toRaw(enc) { var data; - if (this.network && this._raw) + if (this.isStatic() && this._raw) data = this._raw; else data = new Buffer(this.render());