diff --git a/lib/bcoin-browser.js b/lib/bcoin-browser.js index 9240c28d..711818ad 100644 --- a/lib/bcoin-browser.js +++ b/lib/bcoin-browser.js @@ -55,6 +55,13 @@ bcoin.HDPrivateKey = require('./hd/private'); bcoin.HDPublicKey = require('./hd/public'); bcoin.Mnemonic = require('./hd/mnemonic'); +// Index +bcoin.indexer = require('./indexer'); +bcoin.Indexer = require('./indexer/indexer'); +bcoin.ChainClient = require('./indexer/chainclient'); +bcoin.TXIndexer = require('./indexer/txindexer'); +bcoin.AddrIndexer = require('./indexer/addrindexer'); + // Mempool bcoin.mempool = require('./mempool'); bcoin.Fees = require('./mempool/fees'); diff --git a/lib/bcoin.js b/lib/bcoin.js index 72ab240b..8bff0442 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -76,6 +76,13 @@ bcoin.define('HDPrivateKey', './hd/private'); bcoin.define('HDPublicKey', './hd/public'); bcoin.define('Mnemonic', './hd/mnemonic'); +// Index +bcoin.define('indexer', './indexer'); +bcoin.define('Indexer', './indexer/indexer'); +bcoin.define('ChainClient', './indexer/chainclient'); +bcoin.define('TXIndexer', './indexer/txindexer'); +bcoin.define('AddrIndexer', './indexer/addrindexer'); + // Mempool bcoin.define('mempool', './mempool'); bcoin.define('Fees', './mempool/fees'); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 026acb9a..19cfea93 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2059,14 +2059,14 @@ class Chain extends AsyncEmitter { /** * Get coin viewpoint (spent). - * @param {TX} tx + * @param {TXMeta} meta * @returns {Promise} - Returns {@link CoinView}. */ - async getSpentView(tx) { + async getSpentView(meta) { const unlock = await this.locker.lock(); try { - return await this.db.getSpentView(tx); + return await this.db.getSpentView(meta); } finally { unlock(); } @@ -2766,11 +2766,6 @@ class ChainOptions { this.compression = options.compression; } - if (options.prune != null) { - assert(typeof options.prune === 'boolean'); - this.prune = options.prune; - } - if (options.indexTX != null) { assert(typeof options.indexTX === 'boolean'); this.indexTX = options.indexTX; @@ -2781,6 +2776,11 @@ class ChainOptions { this.indexAddress = options.indexAddress; } + if (options.prune != null) { + assert(typeof options.prune === 'boolean'); + this.prune = options.prune; + } + if (options.forceFlags != null) { assert(typeof options.forceFlags === 'boolean'); this.forceFlags = options.forceFlags; diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index f24e6f97..9f7a779e 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -11,7 +11,7 @@ const assert = require('bsert'); const bdb = require('bdb'); const bio = require('bufio'); const LRU = require('blru'); -const {BufferMap, BufferSet} = require('buffer-map'); +const {BufferMap} = require('buffer-map'); const Amount = require('../btc/amount'); const Network = require('../protocol/network'); const CoinView = require('../coins/coinview'); @@ -20,9 +20,7 @@ const layout = require('./layout'); const consensus = require('../protocol/consensus'); const Block = require('../primitives/block'); const Outpoint = require('../primitives/outpoint'); -const Address = require('../primitives/address'); const ChainEntry = require('./chainentry'); -const TXMeta = require('../primitives/txmeta'); const CoinEntry = require('../coins/coinentry'); /** @@ -573,18 +571,12 @@ class ChainDB { if (!options.prune && flags.prune) throw new Error('Cannot retroactively unprune.'); - if (options.indexTX && !flags.indexTX) + if (options.prune && options.indexTX && !flags.indexTX) throw new Error('Cannot retroactively enable TX indexing.'); - if (!options.indexTX && flags.indexTX) - throw new Error('Cannot retroactively disable TX indexing.'); - - if (options.indexAddress && !flags.indexAddress) + if (options.prune && options.indexAddress && !flags.indexAddress) throw new Error('Cannot retroactively enable address indexing.'); - if (!options.indexAddress && flags.indexAddress) - throw new Error('Cannot retroactively disable address indexing.'); - if (needsSave) { await this.logger.info('Rewriting chain flags.'); await this.saveFlags(); @@ -978,30 +970,16 @@ class ChainDB { /** * Get coin viewpoint (historical). - * @param {TX} tx + * @param {TXMeta} meta * @returns {Promise} - Returns {@link CoinView}. */ - async getSpentView(tx) { - const view = await this.getCoinView(tx); - - for (const {prevout} of tx.inputs) { - if (view.hasEntry(prevout)) - continue; - - const {hash, index} = prevout; - const meta = await this.getMeta(hash); - - if (!meta) - continue; - - const {tx, height} = meta; - - if (index < tx.outputs.length) - view.addIndex(tx, index, height); - } - - return view; + async getSpentView(meta) { + process.emitWarning( + 'deprecated, use node.txindex.getSpentView', + 'DeprecationWarning' + ); + return null; } /** @@ -1083,152 +1061,105 @@ class ChainDB { /** * Get a transaction with metadata. * @param {Hash} hash + * @deprecated * @returns {Promise} - Returns {@link TXMeta}. */ async getMeta(hash) { - if (!this.options.indexTX) - return null; - - const data = await this.db.get(layout.t.encode(hash)); - - if (!data) - return null; - - return TXMeta.fromRaw(data); + process.emitWarning( + 'deprecated, use node.txindex.getMeta', + 'DeprecationWarning' + ); + return null; } /** * Retrieve a transaction. * @param {Hash} hash + * @deprecated * @returns {Promise} - Returns {@link TX}. */ async getTX(hash) { - const meta = await this.getMeta(hash); - - if (!meta) - return null; - - return meta.tx; + process.emitWarning( + 'deprecated, use node.txindex.getTX', + 'DeprecationWarning' + ); + return null; } /** * @param {Hash} hash + * @deprecated * @returns {Promise} - Returns Boolean. */ async hasTX(hash) { - if (!this.options.indexTX) - return false; - - return this.db.has(layout.t.encode(hash)); + process.emitWarning( + 'deprecated, use node.txindex.hasTX', + 'DeprecationWarning' + ); + return false; } /** * Get all coins pertinent to an address. * @param {Address[]} addrs + * @deprecated * @returns {Promise} - Returns {@link Coin}[]. */ async getCoinsByAddress(addrs) { - if (!this.options.indexAddress) - return []; - - if (!Array.isArray(addrs)) - addrs = [addrs]; - - const coins = []; - - for (const addr of addrs) { - const hash = Address.getHash(addr); - - const keys = await this.db.keys({ - gte: layout.C.min(hash), - lte: layout.C.max(hash), - parse: (key) => { - const [, txid, index] = layout.C.decode(key); - return [txid, index]; - } - }); - - for (const [hash, index] of keys) { - const coin = await this.getCoin(hash, index); - assert(coin); - coins.push(coin); - } - } - - return coins; + process.emitWarning( + 'deprecated, use node.addrindex.getCoinsByAddress', + 'DeprecationWarning' + ); + return []; } /** * Get all transaction hashes to an address. * @param {Address[]} addrs + * @deprecated * @returns {Promise} - Returns {@link Hash}[]. */ async getHashesByAddress(addrs) { - if (!this.options.indexTX || !this.options.indexAddress) - return []; - - const set = new BufferSet(); - - for (const addr of addrs) { - const hash = Address.getHash(addr); - - await this.db.keys({ - gte: layout.T.min(hash), - lte: layout.T.max(hash), - parse: (key) => { - const [, txid] = layout.T.decode(key); - set.add(txid); - } - }); - } - - return set.toArray(); + process.emitWarning( + 'deprecated, use node.addrindex.getHashesByAddress', + 'DeprecationWarning' + ); + return []; } /** * Get all transactions pertinent to an address. * @param {Address[]} addrs + * @deprecated * @returns {Promise} - Returns {@link TX}[]. */ async getTXByAddress(addrs) { - const mtxs = await this.getMetaByAddress(addrs); - const out = []; - - for (const mtx of mtxs) - out.push(mtx.tx); - - return out; + process.emitWarning( + 'deprecated, use node.addrindex.getHashesByAddress', + 'DeprecationWarning' + ); + return []; } /** * Get all transactions pertinent to an address. * @param {Address[]} addrs + * @deprecated * @returns {Promise} - Returns {@link TXMeta}[]. */ async getMetaByAddress(addrs) { - if (!this.options.indexTX || !this.options.indexAddress) - return []; - - if (!Array.isArray(addrs)) - addrs = [addrs]; - - const hashes = await this.getHashesByAddress(addrs); - const mtxs = []; - - for (const hash of hashes) { - const mtx = await this.getMeta(hash); - assert(mtx); - mtxs.push(mtx); - } - - return mtxs; + process.emitWarning( + 'deprecated, use node.addrindex.getMetaByAddress', + 'DeprecationWarning' + ); + return []; } /** @@ -1771,9 +1702,6 @@ class ChainDB { this.pending.add(output); } - - // Index the transaction if enabled. - this.indexTX(tx, view, entry, i); } // Commit new coin state. @@ -1828,9 +1756,6 @@ class ChainDB { this.pending.spend(output); } - - // Remove from transaction index. - this.unindexTX(tx, view); } // Undo coins should be empty. @@ -1882,105 +1807,6 @@ class ChainDB { b.put(layout.O.encode(), flags.toRaw()); return b.write(); } - - /** - * Index a transaction by txid and address. - * @private - * @param {TX} tx - * @param {CoinView} view - * @param {ChainEntry} entry - * @param {Number} index - */ - - indexTX(tx, view, entry, index) { - const hash = tx.hash(); - - if (this.options.indexTX) { - const meta = TXMeta.fromTX(tx, entry, index); - - this.put(layout.t.encode(hash), meta.toRaw()); - - if (this.options.indexAddress) { - for (const addr of tx.getHashes(view)) - this.put(layout.T.encode(addr, hash), null); - } - } - - if (!this.options.indexAddress) - return; - - if (!tx.isCoinbase()) { - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - const coin = view.getOutput(prevout); - assert(coin); - - const addr = coin.getHash(); - - if (!addr) - continue; - - this.del(layout.C.encode(addr, hash, index)); - } - } - - for (let i = 0; i < tx.outputs.length; i++) { - const output = tx.outputs[i]; - const addr = output.getHash(); - - if (!addr) - continue; - - this.put(layout.C.encode(addr, hash, i), null); - } - } - - /** - * Remove transaction from index. - * @private - * @param {TX} tx - * @param {CoinView} view - */ - - unindexTX(tx, view) { - const hash = tx.hash(); - - if (this.options.indexTX) { - this.del(layout.t.encode(hash)); - if (this.options.indexAddress) { - for (const addr of tx.getHashes(view)) - this.del(layout.T.encode(addr, hash)); - } - } - - if (!this.options.indexAddress) - return; - - if (!tx.isCoinbase()) { - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - const coin = view.getOutput(prevout); - assert(coin); - - const addr = coin.getHash(); - - if (!addr) - continue; - - this.put(layout.C.encode(addr, hash, index), null); - } - } - - for (let i = 0; i < tx.outputs.length; i++) { - const output = tx.outputs[i]; - const addr = output.getHash(); - - if (!addr) - continue; - - this.del(layout.C.encode(addr, hash, i)); - } - } } /** diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 01aaa086..337f9590 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -20,12 +20,12 @@ const bdb = require('bdb'); * n[hash] -> next hash * p[hash] -> tip index * b[hash] -> block (deprecated) - * t[hash] -> extended tx + * t[hash] -> extended tx (deprecated) * c[hash] -> coins * u[hash] -> undo coins (deprecated) * v[bit][hash] -> versionbits state - * T[addr-hash][hash] -> dummy (tx by address) - * C[addr-hash][hash][index] -> dummy (coin by address) + * T[addr-hash][hash] -> dummy (tx by address) (deprecated) + * C[addr-hash][hash][index] -> dummy (coin by address) (deprecated) */ const layout = { diff --git a/lib/indexer/addrindexer.js b/lib/indexer/addrindexer.js new file mode 100644 index 00000000..6d274328 --- /dev/null +++ b/lib/indexer/addrindexer.js @@ -0,0 +1,197 @@ +/*! + * addrindexer.js - addr indexer + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('assert'); +const bdb = require('bdb'); +const {BufferSet} = require('buffer-map'); +const layout = require('./layout'); +const Address = require('../primitives/address'); +const Indexer = require('./indexer'); + +/* + * AddrIndexer Database Layout: + * T[addr-hash][hash] -> dummy (tx by address) + * C[addr-hash][hash][index] -> dummy (coin by address) +*/ + +Object.assign(layout, { + T: bdb.key('T', ['hash', 'hash256']), + C: bdb.key('C', ['hash', 'hash256', 'uint32']) +}); + +/** + * AddrIndexer + * @alias module:indexer.AddrIndexer + * @extends Indexer + */ + +class AddrIndexer extends Indexer { + /** + * Create a indexer + * @constructor + * @param {Object} options + */ + + constructor(options) { + super('addr', options); + + this.db = bdb.create(this.options); + } + + /** + * Index transactions by address. + * @private + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view + */ + + async indexBlock(entry, block, view) { + const b = this.db.batch(); + + for (let i = 0; i < block.txs.length; i++) { + const tx = block.txs[i]; + const hash = tx.hash(); + for (const addr of tx.getHashes(view)) + b.put(layout.T.encode(addr, hash), null); + + if (!tx.isCoinbase()) { + for (const {prevout} of tx.inputs) { + const {hash, index} = prevout; + const coin = view.getOutput(prevout); + assert(coin); + + const addr = coin.getHash(); + + if (!addr) + continue; + + b.del(layout.C.encode(addr, hash, index)); + } + } + + for (let i = 0; i < tx.outputs.length; i++) { + const output = tx.outputs[i]; + const addr = output.getHash(); + + if (!addr) + continue; + + b.put(layout.C.encode(addr, hash, i), null); + } + } + + return b.write(); + } + + /** + * Remove addresses from index. + * @private + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view + */ + + async unindexBlock(entry, block, view) { + const b = this.db.batch(); + for (let i = 0; i < block.txs.length; i++) { + const tx = block.txs[i]; + const hash = tx.hash(); + for (const addr of tx.getHashes(view)) + b.del(layout.T.encode(addr, hash)); + + if (!tx.isCoinbase()) { + for (const {prevout} of tx.inputs) { + const {hash, index} = prevout; + const coin = view.getOutput(prevout); + assert(coin); + + const addr = coin.getHash(); + + if (!addr) + continue; + + b.put(layout.C.encode(addr, hash, index), null); + } + } + + for (let i = 0; i < tx.outputs.length; i++) { + const output = tx.outputs[i]; + const addr = output.getHash(); + + if (!addr) + continue; + + b.del(layout.C.encode(addr, hash, i)); + } + } + + return b.write(); + } + + /** + * Get all coins pertinent to an address. + * @param {Address[]} addrs + * @returns {Promise} - Returns {@link Coin}[]. + */ + + async getCoinsByAddress(addrs) { + if (!Array.isArray(addrs)) + addrs = [addrs]; + + const coins = []; + + for (const addr of addrs) { + const hash = Address.getHash(addr); + + const keys = await this.db.keys({ + gte: layout.C.min(hash), + lte: layout.C.max(hash), + parse: (key) => { + const [, txid, index] = layout.C.decode(key); + return [txid, index]; + } + }); + + for (const [hash, index] of keys) { + const coin = await this.client.getCoin(hash, index); + assert(coin); + coins.push(coin); + } + } + + return coins; + } + + /** + * Get all transaction hashes to an address. + * @param {Address[]} addrs + * @returns {Promise} - Returns {@link Hash}[]. + */ + + async getHashesByAddress(addrs) { + const set = new BufferSet(); + + for (const addr of addrs) { + const hash = Address.getHash(addr); + + await this.db.keys({ + gte: layout.T.min(hash), + lte: layout.T.max(hash), + parse: (key) => { + const [, txid] = layout.T.decode(key); + set.add(txid); + } + }); + } + + return set.toArray(); + } +} + +module.exports = AddrIndexer; diff --git a/lib/indexer/chainclient.js b/lib/indexer/chainclient.js new file mode 100644 index 00000000..cd86d25b --- /dev/null +++ b/lib/indexer/chainclient.js @@ -0,0 +1,200 @@ +/*! + * chainclient.js - chain client for bcoin + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('assert'); +const AsyncEmitter = require('bevent'); +const Chain = require('../blockchain/chain'); + +/** + * Chain Client + * @extends AsyncEmitter + * @alias module:indexer.ChainClient + */ + +class ChainClient extends AsyncEmitter { + /** + * Create a chain client. + * @constructor + * @param {Chain} chain + */ + + constructor(chain) { + super(); + + assert(chain instanceof Chain); + + this.chain = chain; + this.network = chain.network; + this.opened = false; + + this.init(); + } + + /** + * Initialize the client. + */ + + init() { + this.chain.on('connect', async (entry, block, view) => { + if (!this.opened) + return; + + await this.emitAsync('block connect', entry, block, view); + }); + + this.chain.on('disconnect', async (entry, block, view) => { + if (!this.opened) + return; + + await this.emitAsync('block disconnect', entry, block, view); + }); + + this.chain.on('reset', async (tip) => { + if (!this.opened) + return; + + await this.emitAsync('chain reset', tip); + }); + } + + /** + * Open the client. + * @returns {Promise} + */ + + async open(options) { + assert(!this.opened, 'ChainClient is already open.'); + this.opened = true; + setImmediate(() => this.emit('connect')); + } + + /** + * Close the client. + * @returns {Promise} + */ + + async close() { + assert(this.opened, 'ChainClient is not open.'); + this.opened = false; + setImmediate(() => this.emit('disconnect')); + } + + /** + * Get chain tip. + * @returns {Promise} + */ + + async getTip() { + return this.chain.tip; + } + + /** + * Get chain entry. + * @param {Hash} hash + * @returns {Promise} - Returns {@link ChainEntry}. + */ + + async getEntry(hash) { + const entry = await this.chain.getEntry(hash); + + if (!entry) + return null; + + if (!await this.chain.isMainChain(entry)) + return null; + + return entry; + } + + /** + * Get a coin (unspents only). + * @param {Hash} hash + * @param {Number} index + * @returns {Promise} - Returns {@link Coin}. + */ + + async getCoin(hash, index) { + return this.chain.getCoin(hash, index); + } + + /** + * Get hash range. + * @param {Number} start + * @param {Number} end + * @returns {Promise} + */ + + async getHashes(start = -1, end = -1) { + return this.chain.getHashes(start, end); + } + + /** + * Get block + * @param {Hash} hash + * @returns {Promise} - Returns {@link Block} + */ + + async getBlock(hash) { + const block = await this.chain.getBlock(hash); + + if (!block) + return null; + + return block; + } + + /** + * Get a historical block coin viewpoint. + * @param {Block} hash + * @returns {Promise} - Returns {@link CoinView}. + */ + + async getBlockView(block) { + return this.chain.getBlockView(block); + } + + /** + * Get coin viewpoint. + * @param {TX} tx + * @returns {Promise} - Returns {@link CoinView}. + */ + + async getCoinView(tx) { + return this.chain.getCoinView(tx); + } + + /** + * Rescan for any missed blocks. + * @param {Number} start - Start block. + * @returns {Promise} + */ + + async rescan(start) { + for (let i = start; ; i++) { + const entry = await this.getEntry(i); + if (!entry) { + await this.emitAsync('chain tip'); + break; + }; + + const block = await this.getBlock(entry.hash); + assert(block); + + const view = await this.getBlockView(block); + assert(view); + + await this.emitAsync('block rescan', entry, block, view); + } + }; +} + +/* + * Expose + */ + +module.exports = ChainClient; diff --git a/lib/indexer/index.js b/lib/indexer/index.js new file mode 100644 index 00000000..96ad09a3 --- /dev/null +++ b/lib/indexer/index.js @@ -0,0 +1,16 @@ +/*! + * index.js - indexer for bcoin + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +/** + * @module indexer + */ + +exports.Indexer = require('./indexer'); +exports.TXIndexer = require('./txindexer'); +exports.AddrIndexer = require('./addrindexer'); +exports.ChainClient = require('./chainclient'); diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js new file mode 100644 index 00000000..b894d225 --- /dev/null +++ b/lib/indexer/indexer.js @@ -0,0 +1,812 @@ +/*! + * indexer.js - storage for indexes + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('assert'); +const path = require('path'); +const fs = require('bfile'); +const EventEmitter = require('events'); +const {Lock} = require('bmutex'); +const Logger = require('blgr'); +const Network = require('../protocol/network'); +const layout = require('./layout'); +const records = require('./records'); +const ChainClient = require('./chainclient'); +const NullClient = require('./nullclient'); + +const { + ChainState, + BlockMeta +} = records; + +/** + * Indexer + * @alias module:indexer.Indexer + * @extends EventEmitter + * @property {IndexerDB} db + * @property {Number} height + * @property {ChainState} state + * @emits Indexer#chain tip + */ + +class Indexer extends EventEmitter { + /** + * Create a index db. + * @constructor + * @param {String} module + * @param {Object} options + */ + + constructor(module, options) { + super(); + + assert(typeof module === 'string'); + assert(module.length > 0); + + this.options = new IndexOptions(module, options); + + this.network = this.options.network; + this.logger = this.options.logger.context(`${module}indexer`); + this.client = this.options.client || new NullClient(this); + this.db = null; + this.rescanning = false; + + this.state = new ChainState(); + this.height = 0; + + this.lock = new Lock(); + + this.init(); + } + + /** + * Initialize indexdb. + * @private + */ + + init() { + this._bind(); + } + + /** + * Bind to chain events. + * @private + */ + + _bind() { + this.client.on('error', (err) => { + this.emit('error', err); + }); + + this.client.on('connect', async () => { + try { + await this.syncNode(); + } catch (e) { + this.emit('error', e); + } + }); + + this.client.on('block connect', async (entry, block, view) => { + if (this.rescanning) + return; + try { + await this.addBlock(entry, block, view); + } catch (e) { + this.emit('error', e); + } + }); + + this.client.on('block disconnect', async (entry, block, view) => { + if (this.rescanning) + return; + try { + await this.removeBlock(entry, block, view); + } catch (e) { + this.emit('error', e); + } + }); + + this.client.on('block rescan', async (entry, block, view) => { + try { + await this.rescanBlock(entry, block, view); + } catch (e) { + this.emit('error', e); + } + }); + + this.client.on('chain reset', async (tip) => { + try { + await this.resetChain(tip); + } catch (e) { + this.emit('error', e); + } + }); + + this.client.on('chain tip', async () => { + this.logger.debug('Indexer: finished rescan'); + const tip = await this.getTip(); + this.emit('chain tip', tip); + }); + } + + /** + * Ensure prefix directory (prefix/index). + * @returns {Promise} + */ + + async ensure() { + if (fs.unsupported) + return undefined; + + if (this.options.memory) + return undefined; + + return fs.mkdirp(this.options.prefix); + } + + /** + * Open the indexdb, wait for the database to load. + * @returns {Promise} + */ + + async open() { + await this.ensure(); + await this.db.open(); + await this.db.verify(layout.V.encode(), 'index', 0); + + await this.verifyNetwork(); + + await this.connect(); + } + + /** + * Verify network. + * @returns {Promise} + */ + + async verifyNetwork() { + const raw = await this.db.get(layout.O.encode()); + + if (!raw) { + const b = this.db.batch(); + b.put(layout.O.encode(), fromU32(this.network.magic)); + return b.write(); + } + + const magic = raw.readUInt32LE(0, true); + + if (magic !== this.network.magic) + throw new Error('Network mismatch for Indexer.'); + + return undefined; + } + + /** + * Close the indexdb, wait for the database to close. + * @returns {Promise} + */ + + async close() { + await this.disconnect(); + return this.db.close(); + } + + /** + * Connect to the chain server (client required). + * @returns {Promise} + */ + + async connect() { + return this.client.open(); + } + + /** + * Disconnect from chain server (client required). + * @returns {Promise} + */ + + async disconnect() { + return this.client.close(); + } + + /** + * Sync state with server on every connect. + * @returns {Promise} + */ + + async syncNode() { + const unlock = await this.lock.lock(); + try { + this.logger.info('Resyncing from server...'); + await this.syncState(); + await this.syncChain(); + } finally { + unlock(); + } + } + + /** + * Initialize and write initial sync state. + * @returns {Promise} + */ + + async syncState() { + const cache = await this.getState(); + + if (cache) { + this.state = cache; + this.height = cache.height; + + this.logger.info( + 'Indexer loaded (height=%d, start=%d).', + this.state.height, + this.state.startHeight); + return undefined; + } + + this.logger.info('Initializing database state from server.'); + + const b = this.db.batch(); + const hashes = await this.client.getHashes(); + + let tip = null; + + for (let height = 0; height < hashes.length; height++) { + const hash = hashes[height]; + const meta = new BlockMeta(hash, height); + b.put(layout.h.encode(height), meta.toHash()); + tip = meta; + } + + assert(tip); + + const state = this.state.clone(); + state.startHeight = 0; + state.height = tip.height; + + b.put(layout.R.encode(), state.toRaw()); + + await b.write(); + + this.state = state; + this.height = state.height; + + return undefined; + } + + /** + * Connect and sync with the chain server. + * @private + * @returns {Promise} + */ + + async syncChain() { + let height = this.state.height; + + this.logger.info('Syncing state from height %d.', height); + + // re-org when we're offline might + // leave chain in different state. + // scan chain backwards until we + // find a known 'good' height + for (;;) { + const tip = await this.getBlock(height); + assert(tip); + + if (await this.client.getEntry(tip.hash)) + break; + + assert(height !== 0); + height -= 1; + } + + // start scan from last indexed OR + // last known 'good' height whichever + // is lower, because `scan` scans from + // low to high blocks + if (this.state.startHeight < height) + height = this.state.startHeight; + + this.logger.spam('Starting block rescan from: %d.', height); + return this.scan(height); + } + + /** + * Rescan a block. + * @private + * @param {ChainEntry} entry + * @param {TX[]} txs + * @returns {Promise} + */ + + async rescanBlock(entry, block, view) { + this.logger.spam('Rescanning block: %d.', entry.height); + + if (!this.rescanning) { + this.logger.warning('Unsolicited rescan block: %d.', entry.height); + return; + } + + if (entry.height % 1000 === 0) + this.logger.debug('rescanned block: %d.', entry.height); + + if (entry.height > this.state.height + 1) { + this.logger.warning('Rescan block too high: %d.', entry.height); + return; + } + + try { + await this._addBlock(entry, block, view); + } catch (e) { + this.emit('error', e); + throw e; + } + } + + /** + * Rescan blockchain from a given height. + * @private + * @param {Number?} height + * @returns {Promise} + */ + + async scan(height) { + assert((height >>> 0) === height, 'Indexer: Must pass in a height.'); + + await this.rollback(height); + + const tip = this.state.height; + + this.logger.info( + 'Indexer is scanning %d blocks.', + tip - height + 1); + + try { + this.rescanning = true; + this.logger.debug('rescanning from %d to %d', height, tip); + await this.client.rescan(height); + } finally { + this.rescanning = false; + } + } + + /** + * Force a rescan. + * @param {Number} height + * @returns {Promise} + */ + + async rescan(height) { + const unlock = await this.lock.lock(); + try { + return await this._rescan(height); + } finally { + unlock(); + } + } + + /** + * Force a rescan (without a lock). + * @private + * @param {Number} height + * @returns {Promise} + */ + + async _rescan(height) { + return this.scan(height); + } + + /** + * Get the best block hash. + * @returns {Promise} + */ + + async getState() { + const data = await this.db.get(layout.R.encode()); + + if (!data) + return null; + + return ChainState.fromRaw(data); + } + + /** + * Sync the current chain state to tip. + * @param {BlockMeta} tip + * @returns {Promise} + */ + + async setTip(tip) { + const b = this.db.batch(); + const state = this.state.clone(); + + if (tip.height < state.height) { + // Hashes ahead of our new tip + // that we need to delete. + while (state.height !== tip.height) { + b.del(layout.h.encode(state.height)); + state.height -= 1; + } + } else if (tip.height > state.height) { + assert(tip.height === state.height + 1, 'Bad chain sync.'); + state.height += 1; + } + + state.startHeight = tip.height; + + // Save tip and state. + b.put(layout.h.encode(tip.height), tip.toHash()); + b.put(layout.R.encode(), state.toRaw()); + + await b.write(); + + this.state = state; + this.height = state.height; + } + + /** + * Get a index block meta. + * @param {Hash} hash + * @returns {Promise} + */ + + async getBlock(height) { + const data = await this.db.get(layout.h.encode(height)); + + if (!data) + return null; + + const block = new BlockMeta(); + block.hash = data; + block.height = height; + + return block; + } + + /** + * Get index tip. + * @param {Hash} hash + * @returns {Promise} + */ + + async getTip() { + const tip = await this.getBlock(this.state.height); + + if (!tip) + throw new Error('Indexer: Tip not found!'); + + return tip; + } + + /** + * Sync with chain height. + * @param {Number} height + * @returns {Promise} + */ + + async rollback(height) { + if (height > this.state.height) + throw new Error('Indexer: Cannot rollback to the future.'); + + if (height === this.state.height) { + this.logger.info('Rolled back to same height (%d).', height); + return; + } + + this.logger.info( + 'Rolling back %d Indexer blocks to height %d.', + this.state.height - height, height); + + const tip = await this.getBlock(height); + assert(tip); + + await this.revert(tip.height); + await this.setTip(tip); + } + + /** + * Add a block's transactions and write the new best hash. + * @param {ChainEntry} entry + * @param {Block} block + * @returns {Promise} + */ + + async addBlock(entry, block, view) { + const unlock = await this.lock.lock(); + try { + return await this._addBlock(entry, block, view); + } finally { + unlock(); + } + } + + /** + * Add a block's transactions without a lock. + * @private + * @param {ChainEntry} entry + * @param {Block} block + * @returns {Promise} + */ + + async _addBlock(entry, block, view) { + const tip = BlockMeta.fromEntry(entry); + + if (tip.height >= this.network.block.slowHeight && !this.rescanning) + this.logger.debug('Adding block: %d.', tip.height); + + this.logger.spam('Adding block: %d.', entry.height); + + if (tip.height === this.state.height) { + // We let blocks of the same height + // through specifically for rescans: + // we always want to rescan the last + // block since the state may have + // updated before the block was fully + // processed (in the case of a crash). + this.logger.warning('Already saw Indexer block (%d).', tip.height); + } else if (tip.height !== this.state.startHeight + 1) { + await this.scan(this.state.height); + return; + } + + this.logger.spam('Indexing block: %d.', entry.height); + + await this.indexBlock(entry, block, view); + + // Sync the state to the new tip. + await this.setTip(tip); + + return; + } + + /** + * Process block indexing + * Indexers will implement this method to process the block for indexing + * @param {ChainEntry} entry + * @param {Block} block + * @returns {Promise} + */ + + async indexBlock(entry, block, view) { + ; + } + + /** + * Undo block indexing + * Indexers will implement this method to undo indexing for the block + * @param {ChainEntry} entry + * @param {Block} block + * @returns {Promise} + */ + + async unindexBlock(entry, block, view) { + ; + } + + /** + * Revert db to an older state. + * @param {Number} target + * @returns {Promise} + */ + + async revert(target) { + ; + } + + /** + * Unconfirm a block's transactions + * and write the new best hash (SPV version). + * @param {ChainEntry} entry + * @returns {Promise} + */ + + async removeBlock(entry, block, view) { + const unlock = await this.lock.lock(); + try { + return await this._removeBlock(entry, block, view); + } finally { + unlock(); + } + } + + /** + * Unconfirm a block's transactions. + * @private + * @param {ChainEntry} entry + * @returns {Promise} + */ + + async _removeBlock(entry, block, view) { + const tip = BlockMeta.fromEntry(entry); + + this.logger.spam('Removing block: %d.', entry.height); + + if (tip.height === 0) + throw new Error('Indexer: Bad disconnection (genesis block).'); + + if (tip.height > this.state.height) { + this.logger.warning( + 'Indexer is disconnecting high blocks (%d).', + tip.height); + return; + } + + if (tip.height !== this.state.height) + throw new Error('Indexer: Bad disconnection (height mismatch).'); + + this.logger.spam('Unindexing block: %d.', entry.height); + + await this.unindexBlock(entry, block, view); + + const prev = await this.getBlock(tip.height - 1); + assert(prev); + + // Sync the state to the previous tip. + await this.setTip(prev); + + return; + } + + /** + * Handle a chain reset. + * @param {ChainEntry} entry + * @returns {Promise} + */ + + async resetChain(entry) { + const unlock = await this.lock.lock(); + try { + return await this._resetChain(entry); + } finally { + unlock(); + } + } + + /** + * Handle a chain reset without a lock. + * @private + * @param {ChainEntry} entry + * @returns {Promise} + */ + + async _resetChain(entry) { + if (entry.height > this.state.height) + throw new Error('Indexer: Bad reset height.'); + + return this.rollback(entry.height); + } +} + +/** + * Index Options + * @alias module:indexer.IndexOptions + */ + +class IndexOptions { + /** + * Create index options. + * @constructor + * @param {String} module + * @param {Object} options + */ + + constructor(module, options) { + this.module = module; + this.network = Network.primary; + this.logger = Logger.global; + this.client = null; + this.chain = null; + this.indexers = null; + + this.prefix = null; + this.location = null; + this.memory = true; + this.maxFiles = 64; + this.cacheSize = 16 << 20; + this.compression = true; + + if (options) + this.fromOptions(options); + } + + /** + * Inject properties from object. + * @private + * @param {Object} options + * @returns {IndexOptions} + */ + + fromOptions(options) { + if (options.network != null) + this.network = Network.get(options.network); + + if (options.logger != null) { + assert(typeof options.logger === 'object'); + this.logger = options.logger; + } + + if (options.client != null) { + assert(typeof options.client === 'object'); + this.client = options.client; + } + + if (options.chain != null) { + assert(typeof options.chain === 'object'); + this.client = new ChainClient(options.chain); + } + + if (!this.client) { + throw new Error('Client is required'); + } + + if (options.prefix != null) { + assert(typeof options.prefix === 'string'); + this.prefix = options.prefix; + this.prefix = path.join(this.prefix, 'index'); + this.location = path.join(this.prefix, this.module); + } + + if (options.location != null) { + assert(typeof options.location === 'string'); + this.location = options.location; + } + + if (options.memory != null) { + assert(typeof options.memory === 'boolean'); + this.memory = options.memory; + } + + if (options.maxFiles != null) { + assert((options.maxFiles >>> 0) === options.maxFiles); + this.maxFiles = options.maxFiles; + } + + if (options.cacheSize != null) { + assert(Number.isSafeInteger(options.cacheSize) && options.cacheSize >= 0); + this.cacheSize = options.cacheSize; + } + + if (options.compression != null) { + assert(typeof options.compression === 'boolean'); + this.compression = options.compression; + } + + return this; + } + + /** + * Instantiate chain options from object. + * @param {Object} options + * @returns {IndexOptions} + */ + + static fromOptions(options) { + return new this().fromOptions(options); + } +} + +/* + * Helpers + */ + +/** + * fromU32 + * read a 4 byte Uint32LE + * @param {Number} num number + * @returns {Buffer} buffer + */ +function fromU32(num) { + const data = Buffer.allocUnsafe(4); + data.writeUInt32LE(num, 0, true); + return data; +} + +/* + * Expose + */ + +module.exports = Indexer; diff --git a/lib/indexer/layout.js b/lib/indexer/layout.js new file mode 100644 index 00000000..e2bc243c --- /dev/null +++ b/lib/indexer/layout.js @@ -0,0 +1,31 @@ +/*! + * layout.js - indexer layout for bcoin + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const bdb = require('bdb'); + +/* + * Index Database Layout: + * To be extended by indexer implementations + * V -> db version + * O -> flags + * h[height] -> recent block hash + * R -> chain sync state + */ + +const layout = { + V: bdb.key('V'), + O: bdb.key('O'), + h: bdb.key('h', ['uint32']), + R: bdb.key('R') +}; + +/* + * Expose + */ + +module.exports = layout; diff --git a/lib/indexer/nullclient.js b/lib/indexer/nullclient.js new file mode 100644 index 00000000..d71dba5e --- /dev/null +++ b/lib/indexer/nullclient.js @@ -0,0 +1,142 @@ +/*! + * nullclient.js - chain client for bcoin + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('assert'); +const EventEmitter = require('events'); + +/** + * Null Client + * Sort of a fake local client for separation of concerns. + * @alias module:indexer.NullClient + */ + +class NullClient extends EventEmitter { + /** + * Create a client. + * @constructor + * @param {Chain} chain + */ + + constructor(chain) { + super(); + + this.chain = chain; + this.network = chain.network; + this.opened = false; + } + + /** + * Open the client. + * @returns {Promise} + */ + + async open(options) { + assert(!this.opened, 'NullClient is already open.'); + this.opened = true; + setImmediate(() => this.emit('connect')); + } + + /** + * Close the client. + * @returns {Promise} + */ + + async close() { + assert(this.opened, 'NullClient is not open.'); + this.opened = false; + setImmediate(() => this.emit('disconnect')); + } + + /** + * Get chain tip. + * @returns {Promise} + */ + + async getTip() { + const {hash, height, time} = this.network.genesis; + return { hash, height, time }; + } + + /** + * Get chain entry. + * @param {Hash} hash + * @returns {Promise} - Returns {@link ChainEntry}. + */ + + async getEntry(hash) { + return { hash, height: 0, time: 0 }; + } + + /** + * Get a coin (unspents only). + * @param {Hash} hash + * @param {Number} index + * @returns {Promise} - Returns {@link Coin}. + */ + + async getCoin(hash, index) { + return null; + } + + /** + * Get hash range. + * @param {Number} start + * @param {Number} end + * @returns {Promise} + */ + + async getHashes(start = -1, end = -1) { + return [this.network.genesis.hash]; + } + + /** + * Get block + * @param {Hash} hash + * @returns {Promise} + */ + + async getBlock(hash) { + return null; + } + + /** + * Get a historical block coin viewpoint. + * @param {Block} hash + * @returns {Promise} - Returns {@link CoinView}. + */ + + async getBlockView(block) { + return null; + } + + /** + * Get coin viewpoint. + * @param {TX} tx + * @returns {Promise} - Returns {@link CoinView}. + */ + + async getCoinView(tx) { + return null; + } + + /** + * Rescan for any missed blocks. + * @param {Number} start - Start block. + * @returns {Promise} + */ + + async rescan(start) { + ; + } +} + +/* + * Expose + */ + +module.exports = NullClient; diff --git a/lib/indexer/records.js b/lib/indexer/records.js new file mode 100644 index 00000000..3e67e843 --- /dev/null +++ b/lib/indexer/records.js @@ -0,0 +1,221 @@ +/*! + * records.js - indexer records + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +/** + * @module lib/records + */ + +const bio = require('bufio'); +const util = require('../utils/util'); +const consensus = require('../protocol/consensus'); + +/** + * Chain State + * @alias module:indexer.ChainState + */ + +class ChainState { + /** + * Create a chain state. + * @constructor + */ + + constructor() { + this.startHeight = 0; + this.height = 0; + } + + /** + * Clone the state. + * @returns {ChainState} + */ + + clone() { + const state = new ChainState(); + state.startHeight = this.startHeight; + state.height = this.height; + return state; + } + + /** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.startHeight = br.readU32(); + this.height = br.readU32(); + + return this; + } + + /** + * Instantiate chain state from serialized data. + * @param {Buffer} data + * @returns {ChainState} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } + + /** + * Serialize the chain state. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(8); + + bw.writeU32(this.startHeight); + bw.writeU32(this.height); + + return bw.render(); + } +} + +/** + * Block Meta + * @alias module:indexer.BlockMeta + */ + +class BlockMeta { + /** + * Create block meta. + * @constructor + * @param {Hash} hash + * @param {Number} height + */ + + constructor(hash, height) { + this.hash = hash || consensus.NULL_HASH; + this.height = height != null ? height : -1; + } + + /** + * Clone the block. + * @returns {BlockMeta} + */ + + clone() { + return new this.constructor(this.hash, this.height); + } + + /** + * Get block meta hash as a buffer. + * @returns {Buffer} + */ + + toHash() { + return Buffer.from(this.hash, 'hex'); + } + + /** + * Instantiate block meta from chain entry. + * @private + * @param {IndexEntry} entry + */ + + fromEntry(entry) { + this.hash = entry.hash; + this.height = entry.height; + return this; + } + + /** + * Instantiate block meta from json object. + * @private + * @param {Object} json + */ + + fromJSON(json) { + this.hash = util.revHex(json.hash); + this.height = json.height; + return this; + } + + /** + * Instantiate block meta from serialized tip data. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + this.hash = br.readHash('hex'); + this.height = br.readI32(); + return this; + } + + /** + * Instantiate block meta from chain entry. + * @param {IndexEntry} entry + * @returns {BlockMeta} + */ + + static fromEntry(entry) { + return new this().fromEntry(entry); + } + + /** + * Instantiate block meta from json object. + * @param {Object} json + * @returns {BlockMeta} + */ + + static fromJSON(json) { + return new this().fromJSON(json); + } + + /** + * Instantiate block meta from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {BlockMeta} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } + + /** + * Serialize the block meta. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(36); + bw.writeHash(this.hash); + bw.writeI32(this.height); + return bw.render(); + } + + /** + * Convert the block meta to a more json-friendly object. + * @returns {Object} + */ + + toJSON() { + return { + hash: util.revHex(this.hash), + height: this.height + }; + } +} + +/* + * Expose + */ + +exports.ChainState = ChainState; +exports.BlockMeta = BlockMeta; + +module.exports = exports; diff --git a/lib/indexer/txindexer.js b/lib/indexer/txindexer.js new file mode 100644 index 00000000..ad953b63 --- /dev/null +++ b/lib/indexer/txindexer.js @@ -0,0 +1,151 @@ +/*! + * txindexer.js - tx indexer + * Copyright (c) 2018, the bcoin developers (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const bdb = require('bdb'); +const layout = require('./layout'); +const TXMeta = require('../primitives/txmeta'); +const Indexer = require('./indexer'); + +/* + * TXIndexer Database Layout: + * t[hash] -> extended tx +*/ + +Object.assign(layout, { + t: bdb.key('t', ['hash256']) +}); + +/** + * TXIndexer + * @alias module:indexer.TXIndexer + * @extends Indexer + */ + +class TXIndexer extends Indexer { + /** + * Create a indexer + * @constructor + * @param {Object} options + */ + + constructor(options) { + super('tx', options); + + this.db = bdb.create(this.options); + } + + /** + * Index transactions by txid. + * @private + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view + */ + + async indexBlock(entry, block, view) { + const b = this.db.batch(); + + for (let i = 0; i < block.txs.length; i++) { + const tx = block.txs[i]; + const hash = tx.hash(); + const meta = TXMeta.fromTX(tx, entry, i); + b.put(layout.t.encode(hash), meta.toRaw()); + } + + return b.write(); + } + + /** + * Remove transactions from index. + * @private + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view + */ + + async unindexBlock(entry, block, view) { + const b = this.db.batch(); + + for (let i = 0; i < block.txs.length; i++) { + const tx = block.txs[i]; + const hash = tx.hash(); + b.del(layout.t.encode(hash)); + } + + return b.write(); + } + + /** + * Get a transaction with metadata. + * @param {Hash} hash + * @returns {Promise} - Returns {@link TXMeta}. + */ + + async getMeta(hash) { + const data = await this.db.get(layout.t.encode(hash)); + + if (!data) + return null; + + return TXMeta.fromRaw(data); + } + + /** + * Retrieve a transaction. + * @param {Hash} hash + * @returns {Promise} - Returns {@link TX}. + */ + + async getTX(hash) { + const meta = await this.getMeta(hash); + + if (!meta) + return null; + + return meta.tx; + } + + /** + * @param {Hash} hash + * @returns {Promise} - Returns Boolean. + */ + + async hasTX(hash) { + return this.db.has(layout.t.encode(hash)); + } + + /** + * Get coin viewpoint (historical). + * @param {TX} tx + * @returns {Promise} - Returns {@link CoinView}. + */ + + async getSpentView(tx) { + const view = await this.client.getCoinView(tx); + + for (const {prevout} of tx.inputs) { + if (view.hasEntry(prevout)) + continue; + + const {hash, index} = prevout; + const meta = await this.getMeta(hash); + + if (!meta) + continue; + + const {tx, height} = meta; + + if (index < tx.outputs.length) + view.addIndex(tx, index, height); + } + + return view; + } +} + +module.exports = TXIndexer; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 99e1ad9b..6b7c5753 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -17,6 +17,8 @@ const Node = require('./node'); const HTTP = require('./http'); const RPC = require('./rpc'); const blockstore = require('../blockstore'); +const TXIndexer = require('../indexer/txindexer'); +const AddrIndexer = require('../indexer/addrindexer'); /** * Full Node @@ -154,6 +156,27 @@ class FullNode extends Node { cors: this.config.bool('cors') }); + // Indexers + this.txindex = null; + if (this.config.bool('index-tx')) + this.txindex = new TXIndexer({ + network: this.network, + logger: this.logger, + chain: this.chain, + memory: this.config.bool('memory'), + prefix: this.config.filter('index').str('prefix') || this.config.prefix + }); + + this.addrindex = null; + if (this.config.bool('index-address')) + this.addrindex= new AddrIndexer({ + network: this.network, + logger: this.logger, + chain: this.chain, + memory: this.config.bool('memory'), + prefix: this.config.filter('index').str('prefix') || this.config.prefix + }); + this.init(); } @@ -169,6 +192,12 @@ class FullNode extends Node { this.pool.on('error', err => this.error(err)); this.miner.on('error', err => this.error(err)); + if (this.txindex) + this.txindex.on('error', err => this.error(err)); + + if (this.addrindex) + this.addrindex.on('error', err => this.error(err)); + if (this.http) this.http.on('error', err => this.error(err)); @@ -235,6 +264,12 @@ class FullNode extends Node { await this.miner.open(); await this.pool.open(); + if (this.txindex) + await this.txindex.open(); + + if (this.addrindex) + await this.addrindex.open(); + await this.openPlugins(); await this.http.open(); @@ -256,6 +291,12 @@ class FullNode extends Node { await this.handlePreclose(); await this.http.close(); + if (this.txindex) + await this.txindex.close(); + + if (this.addrindex) + await this.addrindex.close(); + await this.closePlugins(); await this.pool.close(); @@ -417,10 +458,14 @@ class FullNode extends Node { async getCoinsByAddress(addrs) { const mempool = this.mempool.getCoinsByAddress(addrs); - const chain = await this.chain.getCoinsByAddress(addrs); + + if (!this.addrindex) + return mempool; + + const index = await this.addrindex.getCoinsByAddress(addrs); const out = []; - for (const coin of chain) { + for (const coin of index) { const spent = this.mempool.isSpent(coin.hash, coin.index); if (spent) @@ -444,8 +489,23 @@ class FullNode extends Node { async getMetaByAddress(addrs) { const mempool = this.mempool.getMetaByAddress(addrs); - const chain = await this.chain.getMetaByAddress(addrs); - return chain.concat(mempool); + + if (this.txindex && this.addrindex) { + if (!Array.isArray(addrs)) + addrs = [addrs]; + + const hashes = await this.addrindex.getHashesByAddress(addrs); + const mtxs = []; + + for (const hash of hashes) { + const mtx = await this.txindex.getMeta(hash); + assert(mtx); + mtxs.push(mtx); + } + return mtxs.concat(mempool); + } + + return mempool; } /** @@ -460,7 +520,10 @@ class FullNode extends Node { if (meta) return meta; - return this.chain.getMeta(hash); + if (this.txindex) + return this.txindex.getMeta(hash); + + return null; } /** @@ -472,7 +535,11 @@ class FullNode extends Node { async getMetaView(meta) { if (meta.height === -1) return this.mempool.getSpentView(meta.tx); - return this.chain.getSpentView(meta.tx); + + if (this.txindex) + return this.txindex.getSpentView(meta.tx); + + return null; } /** @@ -517,7 +584,10 @@ class FullNode extends Node { if (this.mempool.hasEntry(hash)) return true; - return this.chain.hasTX(hash); + if (this.txindex) + return this.txindex.hasTX(hash); + + return false; } } diff --git a/lib/node/node.js b/lib/node/node.js index b407020e..f3d0b7ec 100644 --- a/lib/node/node.js +++ b/lib/node/node.js @@ -64,6 +64,8 @@ class Node extends EventEmitter { this.pool = null; this.miner = null; this.http = null; + this.txindex = null; + this.addrindex = null; this._init(file); } diff --git a/lib/node/rpc.js b/lib/node/rpc.js index d9c08827..90082f11 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -963,8 +963,8 @@ class RPC extends RPCBase { if (hash) { block = await this.chain.getBlock(hash); - } else if (this.chain.options.indexTX) { - const tx = await this.chain.getMeta(last); + } else if (await this.node.hasTX(last)) { + const tx = await this.node.getMeta(last); if (tx) block = await this.chain.getBlock(tx.block); } else { diff --git a/test/indexer-test.js b/test/indexer-test.js new file mode 100644 index 00000000..a6b4ee93 --- /dev/null +++ b/test/indexer-test.js @@ -0,0 +1,129 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const assert = require('./util/assert'); +const reorg = require('./util/reorg'); +const Chain = require('../lib/blockchain/chain'); +const WorkerPool = require('../lib/workers/workerpool'); +const Miner = require('../lib/mining/miner'); +const MemWallet = require('./util/memwallet'); +const TXIndexer = require('../lib/indexer/txindexer'); +const AddrIndexer = require('../lib/indexer/addrindexer'); +const Network = require('../lib/protocol/network'); +const network = Network.get('regtest'); + +const workers = new WorkerPool({ + enabled: true +}); + +const chain = new Chain({ + memory: true, + network, + workers +}); + +const miner = new Miner({ + chain, + version: 4, + workers +}); + +const cpu = miner.cpu; + +const wallet = new MemWallet({ + network +}); + +const txindexer = new TXIndexer({ + 'memory': true, + 'network': network, + 'chain': chain +}); + +const addrindexer = new AddrIndexer({ + 'memory': true, + 'network': network, + 'chain': chain +}); + +describe('Indexer', function() { + this.timeout(45000); + + it('should open indexer', async () => { + await chain.open(); + await miner.open(); + await txindexer.open(); + await addrindexer.open(); + }); + + it('should index 10 blocks', async () => { + miner.addresses.length = 0; + miner.addAddress(wallet.getReceive()); + for (let i = 0; i < 10; i++) { + const block = await cpu.mineBlock(); + assert(block); + assert(await chain.add(block)); + } + + assert.strictEqual(chain.height, 10); + assert.strictEqual(txindexer.state.startHeight, 10); + assert.strictEqual(addrindexer.state.startHeight, 10); + + const coins = + await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 10); + + for (const coin of coins) { + const meta = await txindexer.getMeta(coin.hash); + assert.bufferEqual(meta.tx.hash(), coin.hash); + } + }); + + it('should rescan and reindex 10 missed blocks', async () => { + await txindexer.disconnect(); + await addrindexer.disconnect(); + + for (let i = 0; i < 10; i++) { + const block = await cpu.mineBlock(); + assert(block); + assert(await chain.add(block)); + } + + assert.strictEqual(chain.height, 20); + + await txindexer.connect(); + await addrindexer.connect(); + + await new Promise(r => addrindexer.once('chain tip', r)); + + assert.strictEqual(txindexer.state.startHeight, 20); + assert.strictEqual(addrindexer.state.startHeight, 20); + + const coins = + await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 20); + + for (const coin of coins) { + const meta = await txindexer.getMeta(coin.hash); + assert.bufferEqual(meta.tx.hash(), coin.hash); + } + }); + + it('should handle indexing a reorg', async () => { + await reorg(chain, cpu, 10); + + assert.strictEqual(txindexer.state.startHeight, 31); + assert.strictEqual(addrindexer.state.startHeight, 31); + + const coins = + await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 31); + + for (const coin of coins) { + const meta = await txindexer.getMeta(coin.hash); + assert.bufferEqual(meta.tx.hash(), coin.hash); + } + }); +}); diff --git a/test/node-test.js b/test/node-test.js index 1d7a21bd..fa017fa1 100644 --- a/test/node-test.js +++ b/test/node-test.js @@ -28,6 +28,8 @@ const node = new FullNode({ network: 'regtest', workers: true, plugins: [require('../lib/wallet/plugin')], + indexTX: true, + indexAddress: true, port: ports.p2p, httpPort: ports.node, env: { @@ -756,6 +758,55 @@ describe('Node', function() { assert.strictEqual(tx1.txid(), tx2.txid()); }); + it('should get tx by hash', async () => { + const block = await mineBlock(); + await chain.add(block); + + const tx = block.txs[0]; + const hash = tx.hash(); + const hasTX = await node.hasTX(hash); + + assert.strictEqual(hasTX, true); + + const tx2 = await node.getTX(hash); + assert.strictEqual(tx.txid(), tx2.txid()); + + const meta = await node.getMeta(hash); + assert.strictEqual(meta.tx.txid(), tx2.txid()); + }); + + it('should get coin/tx by addr', async () => { + const addr = await wallet.receiveAddress(); + const mtx = await wallet.createTX({ + rate: 100000, + outputs: [{ + value: 100000, + address: addr + }] + }); + + await wallet.sign(mtx); + + const tx = mtx.toTX(); + const job = await miner.createJob(); + + job.addTX(tx, mtx.view); + job.refresh(); + + const block = await job.mineAsync(); + await chain.add(block); + + await new Promise(r => setTimeout(r, 300)); + + const txs = await node.getTXByAddress(addr.hash); + const tx2 = txs[0]; + assert.strictEqual(tx.txid(), tx2.txid()); + + const coins = await node.getCoinsByAddress(addr.hash); + const coin = coins[0]; + assert.strictEqual(tx.txid(), coin.txid()); + }); + it('should cleanup', async () => { consensus.COINBASE_MATURITY = 100; await node.close(); diff --git a/test/util/reorg.js b/test/util/reorg.js new file mode 100644 index 00000000..bcdf953c --- /dev/null +++ b/test/util/reorg.js @@ -0,0 +1,63 @@ +'use strict'; + +const assert = require('./assert'); +const Chain = require('../../lib/blockchain/chain'); +const CPUMiner = require('../../lib/mining/cpuminer'); + +/** + * Reorgs the chain to given height using miners. + * @param {Chain} chain chain + * @param {CPUMiner} cpu cpuminer + * @param {Number} height height + * @returns {Promise} null + */ +async function reorg(chain, cpu, height) { + assert(chain instanceof Chain); + assert(cpu instanceof CPUMiner); + assert(typeof height === 'number'); + + let tip1, tip2 = null; + for (let i = 0; i < height; i++) { + const job1 = await cpu.createJob(tip1); + const job2 = await cpu.createJob(tip2); + + const blk1 = await job1.mineAsync(); + const blk2 = await job2.mineAsync(); + + const hash1 = blk1.hash(); + const hash2 = blk2.hash(); + + assert(await chain.add(blk1)); + assert(await chain.add(blk2)); + + assert.bufferEqual(chain.tip.hash, hash1); + + tip1 = await chain.getEntry(hash1); + tip2 = await chain.getEntry(hash2); + + assert(tip1); + assert(tip2); + + assert(!await chain.isMainChain(tip2)); + } + + const entry = await chain.getEntry(tip2.hash); + assert(entry); + assert.strictEqual(chain.height, entry.height); + + const block = await cpu.mineBlock(entry); + assert(block); + + let forked = false; + chain.once('reorganize', () => { + forked = true; + }); + + assert(await chain.add(block)); + + assert(forked); + assert.bufferEqual(chain.tip.hash, block.hash()); + assert(chain.tip.chainwork.gt(tip1.chainwork)); +} + +module.exports = reorg;