indexer: add module indexer
module indexer introduces a extensible architecture for indexing the chain. It provides a base class which handles syncing with the chain, handling re-orgs, interruptions, dynamic toggling, etc. TXIndexer and AddrIndexer are provided for indexing transactions and addresses, using the same flags as before i.e --index-tx and --index-address. Indexes are stored in a different database and can be maintained independently of the chain.
This commit is contained in:
Javed Khan
Braydon Fuller
parent
81b840a634
commit
05794f5cb3
@ -55,6 +55,13 @@ bcoin.HDPrivateKey = require('./hd/private');
|
||||
bcoin.HDPublicKey = require('./hd/public');
|
||||
bcoin.Mnemonic = require('./hd/mnemonic');
|
||||
|
||||
// Index
|
||||
bcoin.indexer = require('./indexer');
|
||||
bcoin.Indexer = require('./indexer/indexer');
|
||||
bcoin.ChainClient = require('./indexer/chainclient');
|
||||
bcoin.TXIndexer = require('./indexer/txindexer');
|
||||
bcoin.AddrIndexer = require('./indexer/addrindexer');
|
||||
|
||||
// Mempool
|
||||
bcoin.mempool = require('./mempool');
|
||||
bcoin.Fees = require('./mempool/fees');
|
||||
|
||||
@ -76,6 +76,13 @@ bcoin.define('HDPrivateKey', './hd/private');
|
||||
bcoin.define('HDPublicKey', './hd/public');
|
||||
bcoin.define('Mnemonic', './hd/mnemonic');
|
||||
|
||||
// Index
|
||||
bcoin.define('indexer', './indexer');
|
||||
bcoin.define('Indexer', './indexer/indexer');
|
||||
bcoin.define('ChainClient', './indexer/chainclient');
|
||||
bcoin.define('TXIndexer', './indexer/txindexer');
|
||||
bcoin.define('AddrIndexer', './indexer/addrindexer');
|
||||
|
||||
// Mempool
|
||||
bcoin.define('mempool', './mempool');
|
||||
bcoin.define('Fees', './mempool/fees');
|
||||
|
||||
@ -2059,14 +2059,14 @@ class Chain extends AsyncEmitter {
|
||||
|
||||
/**
|
||||
* Get coin viewpoint (spent).
|
||||
* @param {TX} tx
|
||||
* @param {TXMeta} meta
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getSpentView(tx) {
|
||||
async getSpentView(meta) {
|
||||
const unlock = await this.locker.lock();
|
||||
try {
|
||||
return await this.db.getSpentView(tx);
|
||||
return await this.db.getSpentView(meta);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
@ -2766,11 +2766,6 @@ class ChainOptions {
|
||||
this.compression = options.compression;
|
||||
}
|
||||
|
||||
if (options.prune != null) {
|
||||
assert(typeof options.prune === 'boolean');
|
||||
this.prune = options.prune;
|
||||
}
|
||||
|
||||
if (options.indexTX != null) {
|
||||
assert(typeof options.indexTX === 'boolean');
|
||||
this.indexTX = options.indexTX;
|
||||
@ -2781,6 +2776,11 @@ class ChainOptions {
|
||||
this.indexAddress = options.indexAddress;
|
||||
}
|
||||
|
||||
if (options.prune != null) {
|
||||
assert(typeof options.prune === 'boolean');
|
||||
this.prune = options.prune;
|
||||
}
|
||||
|
||||
if (options.forceFlags != null) {
|
||||
assert(typeof options.forceFlags === 'boolean');
|
||||
this.forceFlags = options.forceFlags;
|
||||
|
||||
@ -11,7 +11,7 @@ const assert = require('bsert');
|
||||
const bdb = require('bdb');
|
||||
const bio = require('bufio');
|
||||
const LRU = require('blru');
|
||||
const {BufferMap, BufferSet} = require('buffer-map');
|
||||
const {BufferMap} = require('buffer-map');
|
||||
const Amount = require('../btc/amount');
|
||||
const Network = require('../protocol/network');
|
||||
const CoinView = require('../coins/coinview');
|
||||
@ -20,9 +20,7 @@ const layout = require('./layout');
|
||||
const consensus = require('../protocol/consensus');
|
||||
const Block = require('../primitives/block');
|
||||
const Outpoint = require('../primitives/outpoint');
|
||||
const Address = require('../primitives/address');
|
||||
const ChainEntry = require('./chainentry');
|
||||
const TXMeta = require('../primitives/txmeta');
|
||||
const CoinEntry = require('../coins/coinentry');
|
||||
|
||||
/**
|
||||
@ -573,18 +571,12 @@ class ChainDB {
|
||||
if (!options.prune && flags.prune)
|
||||
throw new Error('Cannot retroactively unprune.');
|
||||
|
||||
if (options.indexTX && !flags.indexTX)
|
||||
if (options.prune && options.indexTX && !flags.indexTX)
|
||||
throw new Error('Cannot retroactively enable TX indexing.');
|
||||
|
||||
if (!options.indexTX && flags.indexTX)
|
||||
throw new Error('Cannot retroactively disable TX indexing.');
|
||||
|
||||
if (options.indexAddress && !flags.indexAddress)
|
||||
if (options.prune && options.indexAddress && !flags.indexAddress)
|
||||
throw new Error('Cannot retroactively enable address indexing.');
|
||||
|
||||
if (!options.indexAddress && flags.indexAddress)
|
||||
throw new Error('Cannot retroactively disable address indexing.');
|
||||
|
||||
if (needsSave) {
|
||||
await this.logger.info('Rewriting chain flags.');
|
||||
await this.saveFlags();
|
||||
@ -978,30 +970,16 @@ class ChainDB {
|
||||
|
||||
/**
|
||||
* Get coin viewpoint (historical).
|
||||
* @param {TX} tx
|
||||
* @param {TXMeta} meta
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getSpentView(tx) {
|
||||
const view = await this.getCoinView(tx);
|
||||
|
||||
for (const {prevout} of tx.inputs) {
|
||||
if (view.hasEntry(prevout))
|
||||
continue;
|
||||
|
||||
const {hash, index} = prevout;
|
||||
const meta = await this.getMeta(hash);
|
||||
|
||||
if (!meta)
|
||||
continue;
|
||||
|
||||
const {tx, height} = meta;
|
||||
|
||||
if (index < tx.outputs.length)
|
||||
view.addIndex(tx, index, height);
|
||||
}
|
||||
|
||||
return view;
|
||||
async getSpentView(meta) {
|
||||
process.emitWarning(
|
||||
'deprecated, use node.txindex.getSpentView',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1083,152 +1061,105 @@ class ChainDB {
|
||||
/**
|
||||
* Get a transaction with metadata.
|
||||
* @param {Hash} hash
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns {@link TXMeta}.
|
||||
*/
|
||||
|
||||
async getMeta(hash) {
|
||||
if (!this.options.indexTX)
|
||||
return null;
|
||||
|
||||
const data = await this.db.get(layout.t.encode(hash));
|
||||
|
||||
if (!data)
|
||||
return null;
|
||||
|
||||
return TXMeta.fromRaw(data);
|
||||
process.emitWarning(
|
||||
'deprecated, use node.txindex.getMeta',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a transaction.
|
||||
* @param {Hash} hash
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns {@link TX}.
|
||||
*/
|
||||
|
||||
async getTX(hash) {
|
||||
const meta = await this.getMeta(hash);
|
||||
|
||||
if (!meta)
|
||||
return null;
|
||||
|
||||
return meta.tx;
|
||||
process.emitWarning(
|
||||
'deprecated, use node.txindex.getTX',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Hash} hash
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns Boolean.
|
||||
*/
|
||||
|
||||
async hasTX(hash) {
|
||||
if (!this.options.indexTX)
|
||||
return false;
|
||||
|
||||
return this.db.has(layout.t.encode(hash));
|
||||
process.emitWarning(
|
||||
'deprecated, use node.txindex.hasTX',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all coins pertinent to an address.
|
||||
* @param {Address[]} addrs
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns {@link Coin}[].
|
||||
*/
|
||||
|
||||
async getCoinsByAddress(addrs) {
|
||||
if (!this.options.indexAddress)
|
||||
return [];
|
||||
|
||||
if (!Array.isArray(addrs))
|
||||
addrs = [addrs];
|
||||
|
||||
const coins = [];
|
||||
|
||||
for (const addr of addrs) {
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
const keys = await this.db.keys({
|
||||
gte: layout.C.min(hash),
|
||||
lte: layout.C.max(hash),
|
||||
parse: (key) => {
|
||||
const [, txid, index] = layout.C.decode(key);
|
||||
return [txid, index];
|
||||
}
|
||||
});
|
||||
|
||||
for (const [hash, index] of keys) {
|
||||
const coin = await this.getCoin(hash, index);
|
||||
assert(coin);
|
||||
coins.push(coin);
|
||||
}
|
||||
}
|
||||
|
||||
return coins;
|
||||
process.emitWarning(
|
||||
'deprecated, use node.addrindex.getCoinsByAddress',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all transaction hashes to an address.
|
||||
* @param {Address[]} addrs
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns {@link Hash}[].
|
||||
*/
|
||||
|
||||
async getHashesByAddress(addrs) {
|
||||
if (!this.options.indexTX || !this.options.indexAddress)
|
||||
return [];
|
||||
|
||||
const set = new BufferSet();
|
||||
|
||||
for (const addr of addrs) {
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
await this.db.keys({
|
||||
gte: layout.T.min(hash),
|
||||
lte: layout.T.max(hash),
|
||||
parse: (key) => {
|
||||
const [, txid] = layout.T.decode(key);
|
||||
set.add(txid);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return set.toArray();
|
||||
process.emitWarning(
|
||||
'deprecated, use node.addrindex.getHashesByAddress',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all transactions pertinent to an address.
|
||||
* @param {Address[]} addrs
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns {@link TX}[].
|
||||
*/
|
||||
|
||||
async getTXByAddress(addrs) {
|
||||
const mtxs = await this.getMetaByAddress(addrs);
|
||||
const out = [];
|
||||
|
||||
for (const mtx of mtxs)
|
||||
out.push(mtx.tx);
|
||||
|
||||
return out;
|
||||
process.emitWarning(
|
||||
'deprecated, use node.addrindex.getHashesByAddress',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all transactions pertinent to an address.
|
||||
* @param {Address[]} addrs
|
||||
* @deprecated
|
||||
* @returns {Promise} - Returns {@link TXMeta}[].
|
||||
*/
|
||||
|
||||
async getMetaByAddress(addrs) {
|
||||
if (!this.options.indexTX || !this.options.indexAddress)
|
||||
return [];
|
||||
|
||||
if (!Array.isArray(addrs))
|
||||
addrs = [addrs];
|
||||
|
||||
const hashes = await this.getHashesByAddress(addrs);
|
||||
const mtxs = [];
|
||||
|
||||
for (const hash of hashes) {
|
||||
const mtx = await this.getMeta(hash);
|
||||
assert(mtx);
|
||||
mtxs.push(mtx);
|
||||
}
|
||||
|
||||
return mtxs;
|
||||
process.emitWarning(
|
||||
'deprecated, use node.addrindex.getMetaByAddress',
|
||||
'DeprecationWarning'
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1771,9 +1702,6 @@ class ChainDB {
|
||||
|
||||
this.pending.add(output);
|
||||
}
|
||||
|
||||
// Index the transaction if enabled.
|
||||
this.indexTX(tx, view, entry, i);
|
||||
}
|
||||
|
||||
// Commit new coin state.
|
||||
@ -1828,9 +1756,6 @@ class ChainDB {
|
||||
|
||||
this.pending.spend(output);
|
||||
}
|
||||
|
||||
// Remove from transaction index.
|
||||
this.unindexTX(tx, view);
|
||||
}
|
||||
|
||||
// Undo coins should be empty.
|
||||
@ -1882,105 +1807,6 @@ class ChainDB {
|
||||
b.put(layout.O.encode(), flags.toRaw());
|
||||
return b.write();
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a transaction by txid and address.
|
||||
* @private
|
||||
* @param {TX} tx
|
||||
* @param {CoinView} view
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Number} index
|
||||
*/
|
||||
|
||||
indexTX(tx, view, entry, index) {
|
||||
const hash = tx.hash();
|
||||
|
||||
if (this.options.indexTX) {
|
||||
const meta = TXMeta.fromTX(tx, entry, index);
|
||||
|
||||
this.put(layout.t.encode(hash), meta.toRaw());
|
||||
|
||||
if (this.options.indexAddress) {
|
||||
for (const addr of tx.getHashes(view))
|
||||
this.put(layout.T.encode(addr, hash), null);
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.options.indexAddress)
|
||||
return;
|
||||
|
||||
if (!tx.isCoinbase()) {
|
||||
for (const {prevout} of tx.inputs) {
|
||||
const {hash, index} = prevout;
|
||||
const coin = view.getOutput(prevout);
|
||||
assert(coin);
|
||||
|
||||
const addr = coin.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
this.del(layout.C.encode(addr, hash, index));
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < tx.outputs.length; i++) {
|
||||
const output = tx.outputs[i];
|
||||
const addr = output.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
this.put(layout.C.encode(addr, hash, i), null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove transaction from index.
|
||||
* @private
|
||||
* @param {TX} tx
|
||||
* @param {CoinView} view
|
||||
*/
|
||||
|
||||
unindexTX(tx, view) {
|
||||
const hash = tx.hash();
|
||||
|
||||
if (this.options.indexTX) {
|
||||
this.del(layout.t.encode(hash));
|
||||
if (this.options.indexAddress) {
|
||||
for (const addr of tx.getHashes(view))
|
||||
this.del(layout.T.encode(addr, hash));
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.options.indexAddress)
|
||||
return;
|
||||
|
||||
if (!tx.isCoinbase()) {
|
||||
for (const {prevout} of tx.inputs) {
|
||||
const {hash, index} = prevout;
|
||||
const coin = view.getOutput(prevout);
|
||||
assert(coin);
|
||||
|
||||
const addr = coin.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
this.put(layout.C.encode(addr, hash, index), null);
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < tx.outputs.length; i++) {
|
||||
const output = tx.outputs[i];
|
||||
const addr = output.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
this.del(layout.C.encode(addr, hash, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -20,12 +20,12 @@ const bdb = require('bdb');
|
||||
* n[hash] -> next hash
|
||||
* p[hash] -> tip index
|
||||
* b[hash] -> block (deprecated)
|
||||
* t[hash] -> extended tx
|
||||
* t[hash] -> extended tx (deprecated)
|
||||
* c[hash] -> coins
|
||||
* u[hash] -> undo coins (deprecated)
|
||||
* v[bit][hash] -> versionbits state
|
||||
* T[addr-hash][hash] -> dummy (tx by address)
|
||||
* C[addr-hash][hash][index] -> dummy (coin by address)
|
||||
* T[addr-hash][hash] -> dummy (tx by address) (deprecated)
|
||||
* C[addr-hash][hash][index] -> dummy (coin by address) (deprecated)
|
||||
*/
|
||||
|
||||
const layout = {
|
||||
|
||||
197
lib/indexer/addrindexer.js
Normal file
197
lib/indexer/addrindexer.js
Normal file
@ -0,0 +1,197 @@
|
||||
/*!
|
||||
* addrindexer.js - addr indexer
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const bdb = require('bdb');
|
||||
const {BufferSet} = require('buffer-map');
|
||||
const layout = require('./layout');
|
||||
const Address = require('../primitives/address');
|
||||
const Indexer = require('./indexer');
|
||||
|
||||
/*
|
||||
* AddrIndexer Database Layout:
|
||||
* T[addr-hash][hash] -> dummy (tx by address)
|
||||
* C[addr-hash][hash][index] -> dummy (coin by address)
|
||||
*/
|
||||
|
||||
Object.assign(layout, {
|
||||
T: bdb.key('T', ['hash', 'hash256']),
|
||||
C: bdb.key('C', ['hash', 'hash256', 'uint32'])
|
||||
});
|
||||
|
||||
/**
|
||||
* AddrIndexer
|
||||
* @alias module:indexer.AddrIndexer
|
||||
* @extends Indexer
|
||||
*/
|
||||
|
||||
class AddrIndexer extends Indexer {
|
||||
/**
|
||||
* Create a indexer
|
||||
* @constructor
|
||||
* @param {Object} options
|
||||
*/
|
||||
|
||||
constructor(options) {
|
||||
super('addr', options);
|
||||
|
||||
this.db = bdb.create(this.options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Index transactions by address.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @param {CoinView} view
|
||||
*/
|
||||
|
||||
async indexBlock(entry, block, view) {
|
||||
const b = this.db.batch();
|
||||
|
||||
for (let i = 0; i < block.txs.length; i++) {
|
||||
const tx = block.txs[i];
|
||||
const hash = tx.hash();
|
||||
for (const addr of tx.getHashes(view))
|
||||
b.put(layout.T.encode(addr, hash), null);
|
||||
|
||||
if (!tx.isCoinbase()) {
|
||||
for (const {prevout} of tx.inputs) {
|
||||
const {hash, index} = prevout;
|
||||
const coin = view.getOutput(prevout);
|
||||
assert(coin);
|
||||
|
||||
const addr = coin.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
b.del(layout.C.encode(addr, hash, index));
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < tx.outputs.length; i++) {
|
||||
const output = tx.outputs[i];
|
||||
const addr = output.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
b.put(layout.C.encode(addr, hash, i), null);
|
||||
}
|
||||
}
|
||||
|
||||
return b.write();
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove addresses from index.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @param {CoinView} view
|
||||
*/
|
||||
|
||||
async unindexBlock(entry, block, view) {
|
||||
const b = this.db.batch();
|
||||
for (let i = 0; i < block.txs.length; i++) {
|
||||
const tx = block.txs[i];
|
||||
const hash = tx.hash();
|
||||
for (const addr of tx.getHashes(view))
|
||||
b.del(layout.T.encode(addr, hash));
|
||||
|
||||
if (!tx.isCoinbase()) {
|
||||
for (const {prevout} of tx.inputs) {
|
||||
const {hash, index} = prevout;
|
||||
const coin = view.getOutput(prevout);
|
||||
assert(coin);
|
||||
|
||||
const addr = coin.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
b.put(layout.C.encode(addr, hash, index), null);
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < tx.outputs.length; i++) {
|
||||
const output = tx.outputs[i];
|
||||
const addr = output.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
b.del(layout.C.encode(addr, hash, i));
|
||||
}
|
||||
}
|
||||
|
||||
return b.write();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all coins pertinent to an address.
|
||||
* @param {Address[]} addrs
|
||||
* @returns {Promise} - Returns {@link Coin}[].
|
||||
*/
|
||||
|
||||
async getCoinsByAddress(addrs) {
|
||||
if (!Array.isArray(addrs))
|
||||
addrs = [addrs];
|
||||
|
||||
const coins = [];
|
||||
|
||||
for (const addr of addrs) {
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
const keys = await this.db.keys({
|
||||
gte: layout.C.min(hash),
|
||||
lte: layout.C.max(hash),
|
||||
parse: (key) => {
|
||||
const [, txid, index] = layout.C.decode(key);
|
||||
return [txid, index];
|
||||
}
|
||||
});
|
||||
|
||||
for (const [hash, index] of keys) {
|
||||
const coin = await this.client.getCoin(hash, index);
|
||||
assert(coin);
|
||||
coins.push(coin);
|
||||
}
|
||||
}
|
||||
|
||||
return coins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all transaction hashes to an address.
|
||||
* @param {Address[]} addrs
|
||||
* @returns {Promise} - Returns {@link Hash}[].
|
||||
*/
|
||||
|
||||
async getHashesByAddress(addrs) {
|
||||
const set = new BufferSet();
|
||||
|
||||
for (const addr of addrs) {
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
await this.db.keys({
|
||||
gte: layout.T.min(hash),
|
||||
lte: layout.T.max(hash),
|
||||
parse: (key) => {
|
||||
const [, txid] = layout.T.decode(key);
|
||||
set.add(txid);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return set.toArray();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AddrIndexer;
|
||||
200
lib/indexer/chainclient.js
Normal file
200
lib/indexer/chainclient.js
Normal file
@ -0,0 +1,200 @@
|
||||
/*!
|
||||
* chainclient.js - chain client for bcoin
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const AsyncEmitter = require('bevent');
|
||||
const Chain = require('../blockchain/chain');
|
||||
|
||||
/**
|
||||
* Chain Client
|
||||
* @extends AsyncEmitter
|
||||
* @alias module:indexer.ChainClient
|
||||
*/
|
||||
|
||||
class ChainClient extends AsyncEmitter {
|
||||
/**
|
||||
* Create a chain client.
|
||||
* @constructor
|
||||
* @param {Chain} chain
|
||||
*/
|
||||
|
||||
constructor(chain) {
|
||||
super();
|
||||
|
||||
assert(chain instanceof Chain);
|
||||
|
||||
this.chain = chain;
|
||||
this.network = chain.network;
|
||||
this.opened = false;
|
||||
|
||||
this.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the client.
|
||||
*/
|
||||
|
||||
init() {
|
||||
this.chain.on('connect', async (entry, block, view) => {
|
||||
if (!this.opened)
|
||||
return;
|
||||
|
||||
await this.emitAsync('block connect', entry, block, view);
|
||||
});
|
||||
|
||||
this.chain.on('disconnect', async (entry, block, view) => {
|
||||
if (!this.opened)
|
||||
return;
|
||||
|
||||
await this.emitAsync('block disconnect', entry, block, view);
|
||||
});
|
||||
|
||||
this.chain.on('reset', async (tip) => {
|
||||
if (!this.opened)
|
||||
return;
|
||||
|
||||
await this.emitAsync('chain reset', tip);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the client.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async open(options) {
|
||||
assert(!this.opened, 'ChainClient is already open.');
|
||||
this.opened = true;
|
||||
setImmediate(() => this.emit('connect'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the client.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async close() {
|
||||
assert(this.opened, 'ChainClient is not open.');
|
||||
this.opened = false;
|
||||
setImmediate(() => this.emit('disconnect'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get chain tip.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getTip() {
|
||||
return this.chain.tip;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get chain entry.
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise} - Returns {@link ChainEntry}.
|
||||
*/
|
||||
|
||||
async getEntry(hash) {
|
||||
const entry = await this.chain.getEntry(hash);
|
||||
|
||||
if (!entry)
|
||||
return null;
|
||||
|
||||
if (!await this.chain.isMainChain(entry))
|
||||
return null;
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a coin (unspents only).
|
||||
* @param {Hash} hash
|
||||
* @param {Number} index
|
||||
* @returns {Promise} - Returns {@link Coin}.
|
||||
*/
|
||||
|
||||
async getCoin(hash, index) {
|
||||
return this.chain.getCoin(hash, index);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get hash range.
|
||||
* @param {Number} start
|
||||
* @param {Number} end
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getHashes(start = -1, end = -1) {
|
||||
return this.chain.getHashes(start, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get block
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise} - Returns {@link Block}
|
||||
*/
|
||||
|
||||
async getBlock(hash) {
|
||||
const block = await this.chain.getBlock(hash);
|
||||
|
||||
if (!block)
|
||||
return null;
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a historical block coin viewpoint.
|
||||
* @param {Block} hash
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getBlockView(block) {
|
||||
return this.chain.getBlockView(block);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coin viewpoint.
|
||||
* @param {TX} tx
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getCoinView(tx) {
|
||||
return this.chain.getCoinView(tx);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rescan for any missed blocks.
|
||||
* @param {Number} start - Start block.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async rescan(start) {
|
||||
for (let i = start; ; i++) {
|
||||
const entry = await this.getEntry(i);
|
||||
if (!entry) {
|
||||
await this.emitAsync('chain tip');
|
||||
break;
|
||||
};
|
||||
|
||||
const block = await this.getBlock(entry.hash);
|
||||
assert(block);
|
||||
|
||||
const view = await this.getBlockView(block);
|
||||
assert(view);
|
||||
|
||||
await this.emitAsync('block rescan', entry, block, view);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = ChainClient;
|
||||
16
lib/indexer/index.js
Normal file
16
lib/indexer/index.js
Normal file
@ -0,0 +1,16 @@
|
||||
/*!
|
||||
* index.js - indexer for bcoin
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* @module indexer
|
||||
*/
|
||||
|
||||
exports.Indexer = require('./indexer');
|
||||
exports.TXIndexer = require('./txindexer');
|
||||
exports.AddrIndexer = require('./addrindexer');
|
||||
exports.ChainClient = require('./chainclient');
|
||||
812
lib/indexer/indexer.js
Normal file
812
lib/indexer/indexer.js
Normal file
@ -0,0 +1,812 @@
|
||||
/*!
|
||||
* indexer.js - storage for indexes
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const path = require('path');
|
||||
const fs = require('bfile');
|
||||
const EventEmitter = require('events');
|
||||
const {Lock} = require('bmutex');
|
||||
const Logger = require('blgr');
|
||||
const Network = require('../protocol/network');
|
||||
const layout = require('./layout');
|
||||
const records = require('./records');
|
||||
const ChainClient = require('./chainclient');
|
||||
const NullClient = require('./nullclient');
|
||||
|
||||
const {
|
||||
ChainState,
|
||||
BlockMeta
|
||||
} = records;
|
||||
|
||||
/**
|
||||
* Indexer
|
||||
* @alias module:indexer.Indexer
|
||||
* @extends EventEmitter
|
||||
* @property {IndexerDB} db
|
||||
* @property {Number} height
|
||||
* @property {ChainState} state
|
||||
* @emits Indexer#chain tip
|
||||
*/
|
||||
|
||||
class Indexer extends EventEmitter {
|
||||
/**
|
||||
* Create a index db.
|
||||
* @constructor
|
||||
* @param {String} module
|
||||
* @param {Object} options
|
||||
*/
|
||||
|
||||
constructor(module, options) {
|
||||
super();
|
||||
|
||||
assert(typeof module === 'string');
|
||||
assert(module.length > 0);
|
||||
|
||||
this.options = new IndexOptions(module, options);
|
||||
|
||||
this.network = this.options.network;
|
||||
this.logger = this.options.logger.context(`${module}indexer`);
|
||||
this.client = this.options.client || new NullClient(this);
|
||||
this.db = null;
|
||||
this.rescanning = false;
|
||||
|
||||
this.state = new ChainState();
|
||||
this.height = 0;
|
||||
|
||||
this.lock = new Lock();
|
||||
|
||||
this.init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize indexdb.
|
||||
* @private
|
||||
*/
|
||||
|
||||
init() {
|
||||
this._bind();
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind to chain events.
|
||||
* @private
|
||||
*/
|
||||
|
||||
_bind() {
|
||||
this.client.on('error', (err) => {
|
||||
this.emit('error', err);
|
||||
});
|
||||
|
||||
this.client.on('connect', async () => {
|
||||
try {
|
||||
await this.syncNode();
|
||||
} catch (e) {
|
||||
this.emit('error', e);
|
||||
}
|
||||
});
|
||||
|
||||
this.client.on('block connect', async (entry, block, view) => {
|
||||
if (this.rescanning)
|
||||
return;
|
||||
try {
|
||||
await this.addBlock(entry, block, view);
|
||||
} catch (e) {
|
||||
this.emit('error', e);
|
||||
}
|
||||
});
|
||||
|
||||
this.client.on('block disconnect', async (entry, block, view) => {
|
||||
if (this.rescanning)
|
||||
return;
|
||||
try {
|
||||
await this.removeBlock(entry, block, view);
|
||||
} catch (e) {
|
||||
this.emit('error', e);
|
||||
}
|
||||
});
|
||||
|
||||
this.client.on('block rescan', async (entry, block, view) => {
|
||||
try {
|
||||
await this.rescanBlock(entry, block, view);
|
||||
} catch (e) {
|
||||
this.emit('error', e);
|
||||
}
|
||||
});
|
||||
|
||||
this.client.on('chain reset', async (tip) => {
|
||||
try {
|
||||
await this.resetChain(tip);
|
||||
} catch (e) {
|
||||
this.emit('error', e);
|
||||
}
|
||||
});
|
||||
|
||||
this.client.on('chain tip', async () => {
|
||||
this.logger.debug('Indexer: finished rescan');
|
||||
const tip = await this.getTip();
|
||||
this.emit('chain tip', tip);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure prefix directory (prefix/index).
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async ensure() {
|
||||
if (fs.unsupported)
|
||||
return undefined;
|
||||
|
||||
if (this.options.memory)
|
||||
return undefined;
|
||||
|
||||
return fs.mkdirp(this.options.prefix);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the indexdb, wait for the database to load.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async open() {
|
||||
await this.ensure();
|
||||
await this.db.open();
|
||||
await this.db.verify(layout.V.encode(), 'index', 0);
|
||||
|
||||
await this.verifyNetwork();
|
||||
|
||||
await this.connect();
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify network.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async verifyNetwork() {
|
||||
const raw = await this.db.get(layout.O.encode());
|
||||
|
||||
if (!raw) {
|
||||
const b = this.db.batch();
|
||||
b.put(layout.O.encode(), fromU32(this.network.magic));
|
||||
return b.write();
|
||||
}
|
||||
|
||||
const magic = raw.readUInt32LE(0, true);
|
||||
|
||||
if (magic !== this.network.magic)
|
||||
throw new Error('Network mismatch for Indexer.');
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the indexdb, wait for the database to close.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async close() {
|
||||
await this.disconnect();
|
||||
return this.db.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to the chain server (client required).
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async connect() {
|
||||
return this.client.open();
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnect from chain server (client required).
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async disconnect() {
|
||||
return this.client.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync state with server on every connect.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async syncNode() {
|
||||
const unlock = await this.lock.lock();
|
||||
try {
|
||||
this.logger.info('Resyncing from server...');
|
||||
await this.syncState();
|
||||
await this.syncChain();
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize and write initial sync state.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async syncState() {
|
||||
const cache = await this.getState();
|
||||
|
||||
if (cache) {
|
||||
this.state = cache;
|
||||
this.height = cache.height;
|
||||
|
||||
this.logger.info(
|
||||
'Indexer loaded (height=%d, start=%d).',
|
||||
this.state.height,
|
||||
this.state.startHeight);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
this.logger.info('Initializing database state from server.');
|
||||
|
||||
const b = this.db.batch();
|
||||
const hashes = await this.client.getHashes();
|
||||
|
||||
let tip = null;
|
||||
|
||||
for (let height = 0; height < hashes.length; height++) {
|
||||
const hash = hashes[height];
|
||||
const meta = new BlockMeta(hash, height);
|
||||
b.put(layout.h.encode(height), meta.toHash());
|
||||
tip = meta;
|
||||
}
|
||||
|
||||
assert(tip);
|
||||
|
||||
const state = this.state.clone();
|
||||
state.startHeight = 0;
|
||||
state.height = tip.height;
|
||||
|
||||
b.put(layout.R.encode(), state.toRaw());
|
||||
|
||||
await b.write();
|
||||
|
||||
this.state = state;
|
||||
this.height = state.height;
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect and sync with the chain server.
|
||||
* @private
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async syncChain() {
|
||||
let height = this.state.height;
|
||||
|
||||
this.logger.info('Syncing state from height %d.', height);
|
||||
|
||||
// re-org when we're offline might
|
||||
// leave chain in different state.
|
||||
// scan chain backwards until we
|
||||
// find a known 'good' height
|
||||
for (;;) {
|
||||
const tip = await this.getBlock(height);
|
||||
assert(tip);
|
||||
|
||||
if (await this.client.getEntry(tip.hash))
|
||||
break;
|
||||
|
||||
assert(height !== 0);
|
||||
height -= 1;
|
||||
}
|
||||
|
||||
// start scan from last indexed OR
|
||||
// last known 'good' height whichever
|
||||
// is lower, because `scan` scans from
|
||||
// low to high blocks
|
||||
if (this.state.startHeight < height)
|
||||
height = this.state.startHeight;
|
||||
|
||||
this.logger.spam('Starting block rescan from: %d.', height);
|
||||
return this.scan(height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rescan a block.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @param {TX[]} txs
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async rescanBlock(entry, block, view) {
|
||||
this.logger.spam('Rescanning block: %d.', entry.height);
|
||||
|
||||
if (!this.rescanning) {
|
||||
this.logger.warning('Unsolicited rescan block: %d.', entry.height);
|
||||
return;
|
||||
}
|
||||
|
||||
if (entry.height % 1000 === 0)
|
||||
this.logger.debug('rescanned block: %d.', entry.height);
|
||||
|
||||
if (entry.height > this.state.height + 1) {
|
||||
this.logger.warning('Rescan block too high: %d.', entry.height);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this._addBlock(entry, block, view);
|
||||
} catch (e) {
|
||||
this.emit('error', e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rescan blockchain from a given height.
|
||||
* @private
|
||||
* @param {Number?} height
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async scan(height) {
|
||||
assert((height >>> 0) === height, 'Indexer: Must pass in a height.');
|
||||
|
||||
await this.rollback(height);
|
||||
|
||||
const tip = this.state.height;
|
||||
|
||||
this.logger.info(
|
||||
'Indexer is scanning %d blocks.',
|
||||
tip - height + 1);
|
||||
|
||||
try {
|
||||
this.rescanning = true;
|
||||
this.logger.debug('rescanning from %d to %d', height, tip);
|
||||
await this.client.rescan(height);
|
||||
} finally {
|
||||
this.rescanning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force a rescan.
|
||||
* @param {Number} height
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async rescan(height) {
|
||||
const unlock = await this.lock.lock();
|
||||
try {
|
||||
return await this._rescan(height);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force a rescan (without a lock).
|
||||
* @private
|
||||
* @param {Number} height
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async _rescan(height) {
|
||||
return this.scan(height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the best block hash.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getState() {
|
||||
const data = await this.db.get(layout.R.encode());
|
||||
|
||||
if (!data)
|
||||
return null;
|
||||
|
||||
return ChainState.fromRaw(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync the current chain state to tip.
|
||||
* @param {BlockMeta} tip
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async setTip(tip) {
|
||||
const b = this.db.batch();
|
||||
const state = this.state.clone();
|
||||
|
||||
if (tip.height < state.height) {
|
||||
// Hashes ahead of our new tip
|
||||
// that we need to delete.
|
||||
while (state.height !== tip.height) {
|
||||
b.del(layout.h.encode(state.height));
|
||||
state.height -= 1;
|
||||
}
|
||||
} else if (tip.height > state.height) {
|
||||
assert(tip.height === state.height + 1, 'Bad chain sync.');
|
||||
state.height += 1;
|
||||
}
|
||||
|
||||
state.startHeight = tip.height;
|
||||
|
||||
// Save tip and state.
|
||||
b.put(layout.h.encode(tip.height), tip.toHash());
|
||||
b.put(layout.R.encode(), state.toRaw());
|
||||
|
||||
await b.write();
|
||||
|
||||
this.state = state;
|
||||
this.height = state.height;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a index block meta.
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getBlock(height) {
|
||||
const data = await this.db.get(layout.h.encode(height));
|
||||
|
||||
if (!data)
|
||||
return null;
|
||||
|
||||
const block = new BlockMeta();
|
||||
block.hash = data;
|
||||
block.height = height;
|
||||
|
||||
return block;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get index tip.
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getTip() {
|
||||
const tip = await this.getBlock(this.state.height);
|
||||
|
||||
if (!tip)
|
||||
throw new Error('Indexer: Tip not found!');
|
||||
|
||||
return tip;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync with chain height.
|
||||
* @param {Number} height
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async rollback(height) {
|
||||
if (height > this.state.height)
|
||||
throw new Error('Indexer: Cannot rollback to the future.');
|
||||
|
||||
if (height === this.state.height) {
|
||||
this.logger.info('Rolled back to same height (%d).', height);
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info(
|
||||
'Rolling back %d Indexer blocks to height %d.',
|
||||
this.state.height - height, height);
|
||||
|
||||
const tip = await this.getBlock(height);
|
||||
assert(tip);
|
||||
|
||||
await this.revert(tip.height);
|
||||
await this.setTip(tip);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a block's transactions and write the new best hash.
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async addBlock(entry, block, view) {
|
||||
const unlock = await this.lock.lock();
|
||||
try {
|
||||
return await this._addBlock(entry, block, view);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a block's transactions without a lock.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async _addBlock(entry, block, view) {
|
||||
const tip = BlockMeta.fromEntry(entry);
|
||||
|
||||
if (tip.height >= this.network.block.slowHeight && !this.rescanning)
|
||||
this.logger.debug('Adding block: %d.', tip.height);
|
||||
|
||||
this.logger.spam('Adding block: %d.', entry.height);
|
||||
|
||||
if (tip.height === this.state.height) {
|
||||
// We let blocks of the same height
|
||||
// through specifically for rescans:
|
||||
// we always want to rescan the last
|
||||
// block since the state may have
|
||||
// updated before the block was fully
|
||||
// processed (in the case of a crash).
|
||||
this.logger.warning('Already saw Indexer block (%d).', tip.height);
|
||||
} else if (tip.height !== this.state.startHeight + 1) {
|
||||
await this.scan(this.state.height);
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.spam('Indexing block: %d.', entry.height);
|
||||
|
||||
await this.indexBlock(entry, block, view);
|
||||
|
||||
// Sync the state to the new tip.
|
||||
await this.setTip(tip);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process block indexing
|
||||
* Indexers will implement this method to process the block for indexing
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async indexBlock(entry, block, view) {
|
||||
;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undo block indexing
|
||||
* Indexers will implement this method to undo indexing for the block
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async unindexBlock(entry, block, view) {
|
||||
;
|
||||
}
|
||||
|
||||
/**
|
||||
* Revert db to an older state.
|
||||
* @param {Number} target
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async revert(target) {
|
||||
;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unconfirm a block's transactions
|
||||
* and write the new best hash (SPV version).
|
||||
* @param {ChainEntry} entry
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async removeBlock(entry, block, view) {
|
||||
const unlock = await this.lock.lock();
|
||||
try {
|
||||
return await this._removeBlock(entry, block, view);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unconfirm a block's transactions.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async _removeBlock(entry, block, view) {
|
||||
const tip = BlockMeta.fromEntry(entry);
|
||||
|
||||
this.logger.spam('Removing block: %d.', entry.height);
|
||||
|
||||
if (tip.height === 0)
|
||||
throw new Error('Indexer: Bad disconnection (genesis block).');
|
||||
|
||||
if (tip.height > this.state.height) {
|
||||
this.logger.warning(
|
||||
'Indexer is disconnecting high blocks (%d).',
|
||||
tip.height);
|
||||
return;
|
||||
}
|
||||
|
||||
if (tip.height !== this.state.height)
|
||||
throw new Error('Indexer: Bad disconnection (height mismatch).');
|
||||
|
||||
this.logger.spam('Unindexing block: %d.', entry.height);
|
||||
|
||||
await this.unindexBlock(entry, block, view);
|
||||
|
||||
const prev = await this.getBlock(tip.height - 1);
|
||||
assert(prev);
|
||||
|
||||
// Sync the state to the previous tip.
|
||||
await this.setTip(prev);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a chain reset.
|
||||
* @param {ChainEntry} entry
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async resetChain(entry) {
|
||||
const unlock = await this.lock.lock();
|
||||
try {
|
||||
return await this._resetChain(entry);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a chain reset without a lock.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async _resetChain(entry) {
|
||||
if (entry.height > this.state.height)
|
||||
throw new Error('Indexer: Bad reset height.');
|
||||
|
||||
return this.rollback(entry.height);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Index Options
|
||||
* @alias module:indexer.IndexOptions
|
||||
*/
|
||||
|
||||
class IndexOptions {
|
||||
/**
|
||||
* Create index options.
|
||||
* @constructor
|
||||
* @param {String} module
|
||||
* @param {Object} options
|
||||
*/
|
||||
|
||||
constructor(module, options) {
|
||||
this.module = module;
|
||||
this.network = Network.primary;
|
||||
this.logger = Logger.global;
|
||||
this.client = null;
|
||||
this.chain = null;
|
||||
this.indexers = null;
|
||||
|
||||
this.prefix = null;
|
||||
this.location = null;
|
||||
this.memory = true;
|
||||
this.maxFiles = 64;
|
||||
this.cacheSize = 16 << 20;
|
||||
this.compression = true;
|
||||
|
||||
if (options)
|
||||
this.fromOptions(options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject properties from object.
|
||||
* @private
|
||||
* @param {Object} options
|
||||
* @returns {IndexOptions}
|
||||
*/
|
||||
|
||||
fromOptions(options) {
|
||||
if (options.network != null)
|
||||
this.network = Network.get(options.network);
|
||||
|
||||
if (options.logger != null) {
|
||||
assert(typeof options.logger === 'object');
|
||||
this.logger = options.logger;
|
||||
}
|
||||
|
||||
if (options.client != null) {
|
||||
assert(typeof options.client === 'object');
|
||||
this.client = options.client;
|
||||
}
|
||||
|
||||
if (options.chain != null) {
|
||||
assert(typeof options.chain === 'object');
|
||||
this.client = new ChainClient(options.chain);
|
||||
}
|
||||
|
||||
if (!this.client) {
|
||||
throw new Error('Client is required');
|
||||
}
|
||||
|
||||
if (options.prefix != null) {
|
||||
assert(typeof options.prefix === 'string');
|
||||
this.prefix = options.prefix;
|
||||
this.prefix = path.join(this.prefix, 'index');
|
||||
this.location = path.join(this.prefix, this.module);
|
||||
}
|
||||
|
||||
if (options.location != null) {
|
||||
assert(typeof options.location === 'string');
|
||||
this.location = options.location;
|
||||
}
|
||||
|
||||
if (options.memory != null) {
|
||||
assert(typeof options.memory === 'boolean');
|
||||
this.memory = options.memory;
|
||||
}
|
||||
|
||||
if (options.maxFiles != null) {
|
||||
assert((options.maxFiles >>> 0) === options.maxFiles);
|
||||
this.maxFiles = options.maxFiles;
|
||||
}
|
||||
|
||||
if (options.cacheSize != null) {
|
||||
assert(Number.isSafeInteger(options.cacheSize) && options.cacheSize >= 0);
|
||||
this.cacheSize = options.cacheSize;
|
||||
}
|
||||
|
||||
if (options.compression != null) {
|
||||
assert(typeof options.compression === 'boolean');
|
||||
this.compression = options.compression;
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate chain options from object.
|
||||
* @param {Object} options
|
||||
* @returns {IndexOptions}
|
||||
*/
|
||||
|
||||
static fromOptions(options) {
|
||||
return new this().fromOptions(options);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Helpers
|
||||
*/
|
||||
|
||||
/**
|
||||
* fromU32
|
||||
* read a 4 byte Uint32LE
|
||||
* @param {Number} num number
|
||||
* @returns {Buffer} buffer
|
||||
*/
|
||||
function fromU32(num) {
|
||||
const data = Buffer.allocUnsafe(4);
|
||||
data.writeUInt32LE(num, 0, true);
|
||||
return data;
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = Indexer;
|
||||
31
lib/indexer/layout.js
Normal file
31
lib/indexer/layout.js
Normal file
@ -0,0 +1,31 @@
|
||||
/*!
|
||||
* layout.js - indexer layout for bcoin
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const bdb = require('bdb');
|
||||
|
||||
/*
|
||||
* Index Database Layout:
|
||||
* To be extended by indexer implementations
|
||||
* V -> db version
|
||||
* O -> flags
|
||||
* h[height] -> recent block hash
|
||||
* R -> chain sync state
|
||||
*/
|
||||
|
||||
const layout = {
|
||||
V: bdb.key('V'),
|
||||
O: bdb.key('O'),
|
||||
h: bdb.key('h', ['uint32']),
|
||||
R: bdb.key('R')
|
||||
};
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = layout;
|
||||
142
lib/indexer/nullclient.js
Normal file
142
lib/indexer/nullclient.js
Normal file
@ -0,0 +1,142 @@
|
||||
/*!
|
||||
* nullclient.js - chain client for bcoin
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const EventEmitter = require('events');
|
||||
|
||||
/**
|
||||
* Null Client
|
||||
* Sort of a fake local client for separation of concerns.
|
||||
* @alias module:indexer.NullClient
|
||||
*/
|
||||
|
||||
class NullClient extends EventEmitter {
|
||||
/**
|
||||
* Create a client.
|
||||
* @constructor
|
||||
* @param {Chain} chain
|
||||
*/
|
||||
|
||||
constructor(chain) {
|
||||
super();
|
||||
|
||||
this.chain = chain;
|
||||
this.network = chain.network;
|
||||
this.opened = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the client.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async open(options) {
|
||||
assert(!this.opened, 'NullClient is already open.');
|
||||
this.opened = true;
|
||||
setImmediate(() => this.emit('connect'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the client.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async close() {
|
||||
assert(this.opened, 'NullClient is not open.');
|
||||
this.opened = false;
|
||||
setImmediate(() => this.emit('disconnect'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get chain tip.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getTip() {
|
||||
const {hash, height, time} = this.network.genesis;
|
||||
return { hash, height, time };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get chain entry.
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise} - Returns {@link ChainEntry}.
|
||||
*/
|
||||
|
||||
async getEntry(hash) {
|
||||
return { hash, height: 0, time: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a coin (unspents only).
|
||||
* @param {Hash} hash
|
||||
* @param {Number} index
|
||||
* @returns {Promise} - Returns {@link Coin}.
|
||||
*/
|
||||
|
||||
async getCoin(hash, index) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get hash range.
|
||||
* @param {Number} start
|
||||
* @param {Number} end
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getHashes(start = -1, end = -1) {
|
||||
return [this.network.genesis.hash];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get block
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async getBlock(hash) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a historical block coin viewpoint.
|
||||
* @param {Block} hash
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getBlockView(block) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coin viewpoint.
|
||||
* @param {TX} tx
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getCoinView(tx) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rescan for any missed blocks.
|
||||
* @param {Number} start - Start block.
|
||||
* @returns {Promise}
|
||||
*/
|
||||
|
||||
async rescan(start) {
|
||||
;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = NullClient;
|
||||
221
lib/indexer/records.js
Normal file
221
lib/indexer/records.js
Normal file
@ -0,0 +1,221 @@
|
||||
/*!
|
||||
* records.js - indexer records
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* @module lib/records
|
||||
*/
|
||||
|
||||
const bio = require('bufio');
|
||||
const util = require('../utils/util');
|
||||
const consensus = require('../protocol/consensus');
|
||||
|
||||
/**
|
||||
* Chain State
|
||||
* @alias module:indexer.ChainState
|
||||
*/
|
||||
|
||||
class ChainState {
|
||||
/**
|
||||
* Create a chain state.
|
||||
* @constructor
|
||||
*/
|
||||
|
||||
constructor() {
|
||||
this.startHeight = 0;
|
||||
this.height = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clone the state.
|
||||
* @returns {ChainState}
|
||||
*/
|
||||
|
||||
clone() {
|
||||
const state = new ChainState();
|
||||
state.startHeight = this.startHeight;
|
||||
state.height = this.height;
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject properties from serialized data.
|
||||
* @private
|
||||
* @param {Buffer} data
|
||||
*/
|
||||
|
||||
fromRaw(data) {
|
||||
const br = bio.read(data);
|
||||
|
||||
this.startHeight = br.readU32();
|
||||
this.height = br.readU32();
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate chain state from serialized data.
|
||||
* @param {Buffer} data
|
||||
* @returns {ChainState}
|
||||
*/
|
||||
|
||||
static fromRaw(data) {
|
||||
return new this().fromRaw(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the chain state.
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
|
||||
toRaw() {
|
||||
const bw = bio.write(8);
|
||||
|
||||
bw.writeU32(this.startHeight);
|
||||
bw.writeU32(this.height);
|
||||
|
||||
return bw.render();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Block Meta
|
||||
* @alias module:indexer.BlockMeta
|
||||
*/
|
||||
|
||||
class BlockMeta {
|
||||
/**
|
||||
* Create block meta.
|
||||
* @constructor
|
||||
* @param {Hash} hash
|
||||
* @param {Number} height
|
||||
*/
|
||||
|
||||
constructor(hash, height) {
|
||||
this.hash = hash || consensus.NULL_HASH;
|
||||
this.height = height != null ? height : -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clone the block.
|
||||
* @returns {BlockMeta}
|
||||
*/
|
||||
|
||||
clone() {
|
||||
return new this.constructor(this.hash, this.height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get block meta hash as a buffer.
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
|
||||
toHash() {
|
||||
return Buffer.from(this.hash, 'hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate block meta from chain entry.
|
||||
* @private
|
||||
* @param {IndexEntry} entry
|
||||
*/
|
||||
|
||||
fromEntry(entry) {
|
||||
this.hash = entry.hash;
|
||||
this.height = entry.height;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate block meta from json object.
|
||||
* @private
|
||||
* @param {Object} json
|
||||
*/
|
||||
|
||||
fromJSON(json) {
|
||||
this.hash = util.revHex(json.hash);
|
||||
this.height = json.height;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate block meta from serialized tip data.
|
||||
* @private
|
||||
* @param {Buffer} data
|
||||
*/
|
||||
|
||||
fromRaw(data) {
|
||||
const br = bio.read(data);
|
||||
this.hash = br.readHash('hex');
|
||||
this.height = br.readI32();
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate block meta from chain entry.
|
||||
* @param {IndexEntry} entry
|
||||
* @returns {BlockMeta}
|
||||
*/
|
||||
|
||||
static fromEntry(entry) {
|
||||
return new this().fromEntry(entry);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate block meta from json object.
|
||||
* @param {Object} json
|
||||
* @returns {BlockMeta}
|
||||
*/
|
||||
|
||||
static fromJSON(json) {
|
||||
return new this().fromJSON(json);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiate block meta from serialized data.
|
||||
* @param {Hash} hash
|
||||
* @param {Buffer} data
|
||||
* @returns {BlockMeta}
|
||||
*/
|
||||
|
||||
static fromRaw(data) {
|
||||
return new this().fromRaw(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the block meta.
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
|
||||
toRaw() {
|
||||
const bw = bio.write(36);
|
||||
bw.writeHash(this.hash);
|
||||
bw.writeI32(this.height);
|
||||
return bw.render();
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the block meta to a more json-friendly object.
|
||||
* @returns {Object}
|
||||
*/
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
hash: util.revHex(this.hash),
|
||||
height: this.height
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
exports.ChainState = ChainState;
|
||||
exports.BlockMeta = BlockMeta;
|
||||
|
||||
module.exports = exports;
|
||||
151
lib/indexer/txindexer.js
Normal file
151
lib/indexer/txindexer.js
Normal file
@ -0,0 +1,151 @@
|
||||
/*!
|
||||
* txindexer.js - tx indexer
|
||||
* Copyright (c) 2018, the bcoin developers (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const bdb = require('bdb');
|
||||
const layout = require('./layout');
|
||||
const TXMeta = require('../primitives/txmeta');
|
||||
const Indexer = require('./indexer');
|
||||
|
||||
/*
|
||||
* TXIndexer Database Layout:
|
||||
* t[hash] -> extended tx
|
||||
*/
|
||||
|
||||
Object.assign(layout, {
|
||||
t: bdb.key('t', ['hash256'])
|
||||
});
|
||||
|
||||
/**
|
||||
* TXIndexer
|
||||
* @alias module:indexer.TXIndexer
|
||||
* @extends Indexer
|
||||
*/
|
||||
|
||||
class TXIndexer extends Indexer {
|
||||
/**
|
||||
* Create a indexer
|
||||
* @constructor
|
||||
* @param {Object} options
|
||||
*/
|
||||
|
||||
constructor(options) {
|
||||
super('tx', options);
|
||||
|
||||
this.db = bdb.create(this.options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Index transactions by txid.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @param {CoinView} view
|
||||
*/
|
||||
|
||||
async indexBlock(entry, block, view) {
|
||||
const b = this.db.batch();
|
||||
|
||||
for (let i = 0; i < block.txs.length; i++) {
|
||||
const tx = block.txs[i];
|
||||
const hash = tx.hash();
|
||||
const meta = TXMeta.fromTX(tx, entry, i);
|
||||
b.put(layout.t.encode(hash), meta.toRaw());
|
||||
}
|
||||
|
||||
return b.write();
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove transactions from index.
|
||||
* @private
|
||||
* @param {ChainEntry} entry
|
||||
* @param {Block} block
|
||||
* @param {CoinView} view
|
||||
*/
|
||||
|
||||
async unindexBlock(entry, block, view) {
|
||||
const b = this.db.batch();
|
||||
|
||||
for (let i = 0; i < block.txs.length; i++) {
|
||||
const tx = block.txs[i];
|
||||
const hash = tx.hash();
|
||||
b.del(layout.t.encode(hash));
|
||||
}
|
||||
|
||||
return b.write();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a transaction with metadata.
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise} - Returns {@link TXMeta}.
|
||||
*/
|
||||
|
||||
async getMeta(hash) {
|
||||
const data = await this.db.get(layout.t.encode(hash));
|
||||
|
||||
if (!data)
|
||||
return null;
|
||||
|
||||
return TXMeta.fromRaw(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a transaction.
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise} - Returns {@link TX}.
|
||||
*/
|
||||
|
||||
async getTX(hash) {
|
||||
const meta = await this.getMeta(hash);
|
||||
|
||||
if (!meta)
|
||||
return null;
|
||||
|
||||
return meta.tx;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Hash} hash
|
||||
* @returns {Promise} - Returns Boolean.
|
||||
*/
|
||||
|
||||
async hasTX(hash) {
|
||||
return this.db.has(layout.t.encode(hash));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coin viewpoint (historical).
|
||||
* @param {TX} tx
|
||||
* @returns {Promise} - Returns {@link CoinView}.
|
||||
*/
|
||||
|
||||
async getSpentView(tx) {
|
||||
const view = await this.client.getCoinView(tx);
|
||||
|
||||
for (const {prevout} of tx.inputs) {
|
||||
if (view.hasEntry(prevout))
|
||||
continue;
|
||||
|
||||
const {hash, index} = prevout;
|
||||
const meta = await this.getMeta(hash);
|
||||
|
||||
if (!meta)
|
||||
continue;
|
||||
|
||||
const {tx, height} = meta;
|
||||
|
||||
if (index < tx.outputs.length)
|
||||
view.addIndex(tx, index, height);
|
||||
}
|
||||
|
||||
return view;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = TXIndexer;
|
||||
@ -17,6 +17,8 @@ const Node = require('./node');
|
||||
const HTTP = require('./http');
|
||||
const RPC = require('./rpc');
|
||||
const blockstore = require('../blockstore');
|
||||
const TXIndexer = require('../indexer/txindexer');
|
||||
const AddrIndexer = require('../indexer/addrindexer');
|
||||
|
||||
/**
|
||||
* Full Node
|
||||
@ -154,6 +156,27 @@ class FullNode extends Node {
|
||||
cors: this.config.bool('cors')
|
||||
});
|
||||
|
||||
// Indexers
|
||||
this.txindex = null;
|
||||
if (this.config.bool('index-tx'))
|
||||
this.txindex = new TXIndexer({
|
||||
network: this.network,
|
||||
logger: this.logger,
|
||||
chain: this.chain,
|
||||
memory: this.config.bool('memory'),
|
||||
prefix: this.config.filter('index').str('prefix') || this.config.prefix
|
||||
});
|
||||
|
||||
this.addrindex = null;
|
||||
if (this.config.bool('index-address'))
|
||||
this.addrindex= new AddrIndexer({
|
||||
network: this.network,
|
||||
logger: this.logger,
|
||||
chain: this.chain,
|
||||
memory: this.config.bool('memory'),
|
||||
prefix: this.config.filter('index').str('prefix') || this.config.prefix
|
||||
});
|
||||
|
||||
this.init();
|
||||
}
|
||||
|
||||
@ -169,6 +192,12 @@ class FullNode extends Node {
|
||||
this.pool.on('error', err => this.error(err));
|
||||
this.miner.on('error', err => this.error(err));
|
||||
|
||||
if (this.txindex)
|
||||
this.txindex.on('error', err => this.error(err));
|
||||
|
||||
if (this.addrindex)
|
||||
this.addrindex.on('error', err => this.error(err));
|
||||
|
||||
if (this.http)
|
||||
this.http.on('error', err => this.error(err));
|
||||
|
||||
@ -235,6 +264,12 @@ class FullNode extends Node {
|
||||
await this.miner.open();
|
||||
await this.pool.open();
|
||||
|
||||
if (this.txindex)
|
||||
await this.txindex.open();
|
||||
|
||||
if (this.addrindex)
|
||||
await this.addrindex.open();
|
||||
|
||||
await this.openPlugins();
|
||||
|
||||
await this.http.open();
|
||||
@ -256,6 +291,12 @@ class FullNode extends Node {
|
||||
await this.handlePreclose();
|
||||
await this.http.close();
|
||||
|
||||
if (this.txindex)
|
||||
await this.txindex.close();
|
||||
|
||||
if (this.addrindex)
|
||||
await this.addrindex.close();
|
||||
|
||||
await this.closePlugins();
|
||||
|
||||
await this.pool.close();
|
||||
@ -417,10 +458,14 @@ class FullNode extends Node {
|
||||
|
||||
async getCoinsByAddress(addrs) {
|
||||
const mempool = this.mempool.getCoinsByAddress(addrs);
|
||||
const chain = await this.chain.getCoinsByAddress(addrs);
|
||||
|
||||
if (!this.addrindex)
|
||||
return mempool;
|
||||
|
||||
const index = await this.addrindex.getCoinsByAddress(addrs);
|
||||
const out = [];
|
||||
|
||||
for (const coin of chain) {
|
||||
for (const coin of index) {
|
||||
const spent = this.mempool.isSpent(coin.hash, coin.index);
|
||||
|
||||
if (spent)
|
||||
@ -444,8 +489,23 @@ class FullNode extends Node {
|
||||
|
||||
async getMetaByAddress(addrs) {
|
||||
const mempool = this.mempool.getMetaByAddress(addrs);
|
||||
const chain = await this.chain.getMetaByAddress(addrs);
|
||||
return chain.concat(mempool);
|
||||
|
||||
if (this.txindex && this.addrindex) {
|
||||
if (!Array.isArray(addrs))
|
||||
addrs = [addrs];
|
||||
|
||||
const hashes = await this.addrindex.getHashesByAddress(addrs);
|
||||
const mtxs = [];
|
||||
|
||||
for (const hash of hashes) {
|
||||
const mtx = await this.txindex.getMeta(hash);
|
||||
assert(mtx);
|
||||
mtxs.push(mtx);
|
||||
}
|
||||
return mtxs.concat(mempool);
|
||||
}
|
||||
|
||||
return mempool;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -460,7 +520,10 @@ class FullNode extends Node {
|
||||
if (meta)
|
||||
return meta;
|
||||
|
||||
return this.chain.getMeta(hash);
|
||||
if (this.txindex)
|
||||
return this.txindex.getMeta(hash);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -472,7 +535,11 @@ class FullNode extends Node {
|
||||
async getMetaView(meta) {
|
||||
if (meta.height === -1)
|
||||
return this.mempool.getSpentView(meta.tx);
|
||||
return this.chain.getSpentView(meta.tx);
|
||||
|
||||
if (this.txindex)
|
||||
return this.txindex.getSpentView(meta.tx);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -517,7 +584,10 @@ class FullNode extends Node {
|
||||
if (this.mempool.hasEntry(hash))
|
||||
return true;
|
||||
|
||||
return this.chain.hasTX(hash);
|
||||
if (this.txindex)
|
||||
return this.txindex.hasTX(hash);
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -64,6 +64,8 @@ class Node extends EventEmitter {
|
||||
this.pool = null;
|
||||
this.miner = null;
|
||||
this.http = null;
|
||||
this.txindex = null;
|
||||
this.addrindex = null;
|
||||
|
||||
this._init(file);
|
||||
}
|
||||
|
||||
@ -963,8 +963,8 @@ class RPC extends RPCBase {
|
||||
|
||||
if (hash) {
|
||||
block = await this.chain.getBlock(hash);
|
||||
} else if (this.chain.options.indexTX) {
|
||||
const tx = await this.chain.getMeta(last);
|
||||
} else if (await this.node.hasTX(last)) {
|
||||
const tx = await this.node.getMeta(last);
|
||||
if (tx)
|
||||
block = await this.chain.getBlock(tx.block);
|
||||
} else {
|
||||
|
||||
129
test/indexer-test.js
Normal file
129
test/indexer-test.js
Normal file
@ -0,0 +1,129 @@
|
||||
/* eslint-env mocha */
|
||||
/* eslint prefer-arrow-callback: "off" */
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('./util/assert');
|
||||
const reorg = require('./util/reorg');
|
||||
const Chain = require('../lib/blockchain/chain');
|
||||
const WorkerPool = require('../lib/workers/workerpool');
|
||||
const Miner = require('../lib/mining/miner');
|
||||
const MemWallet = require('./util/memwallet');
|
||||
const TXIndexer = require('../lib/indexer/txindexer');
|
||||
const AddrIndexer = require('../lib/indexer/addrindexer');
|
||||
const Network = require('../lib/protocol/network');
|
||||
const network = Network.get('regtest');
|
||||
|
||||
const workers = new WorkerPool({
|
||||
enabled: true
|
||||
});
|
||||
|
||||
const chain = new Chain({
|
||||
memory: true,
|
||||
network,
|
||||
workers
|
||||
});
|
||||
|
||||
const miner = new Miner({
|
||||
chain,
|
||||
version: 4,
|
||||
workers
|
||||
});
|
||||
|
||||
const cpu = miner.cpu;
|
||||
|
||||
const wallet = new MemWallet({
|
||||
network
|
||||
});
|
||||
|
||||
const txindexer = new TXIndexer({
|
||||
'memory': true,
|
||||
'network': network,
|
||||
'chain': chain
|
||||
});
|
||||
|
||||
const addrindexer = new AddrIndexer({
|
||||
'memory': true,
|
||||
'network': network,
|
||||
'chain': chain
|
||||
});
|
||||
|
||||
describe('Indexer', function() {
|
||||
this.timeout(45000);
|
||||
|
||||
it('should open indexer', async () => {
|
||||
await chain.open();
|
||||
await miner.open();
|
||||
await txindexer.open();
|
||||
await addrindexer.open();
|
||||
});
|
||||
|
||||
it('should index 10 blocks', async () => {
|
||||
miner.addresses.length = 0;
|
||||
miner.addAddress(wallet.getReceive());
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const block = await cpu.mineBlock();
|
||||
assert(block);
|
||||
assert(await chain.add(block));
|
||||
}
|
||||
|
||||
assert.strictEqual(chain.height, 10);
|
||||
assert.strictEqual(txindexer.state.startHeight, 10);
|
||||
assert.strictEqual(addrindexer.state.startHeight, 10);
|
||||
|
||||
const coins =
|
||||
await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 10);
|
||||
|
||||
for (const coin of coins) {
|
||||
const meta = await txindexer.getMeta(coin.hash);
|
||||
assert.bufferEqual(meta.tx.hash(), coin.hash);
|
||||
}
|
||||
});
|
||||
|
||||
it('should rescan and reindex 10 missed blocks', async () => {
|
||||
await txindexer.disconnect();
|
||||
await addrindexer.disconnect();
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const block = await cpu.mineBlock();
|
||||
assert(block);
|
||||
assert(await chain.add(block));
|
||||
}
|
||||
|
||||
assert.strictEqual(chain.height, 20);
|
||||
|
||||
await txindexer.connect();
|
||||
await addrindexer.connect();
|
||||
|
||||
await new Promise(r => addrindexer.once('chain tip', r));
|
||||
|
||||
assert.strictEqual(txindexer.state.startHeight, 20);
|
||||
assert.strictEqual(addrindexer.state.startHeight, 20);
|
||||
|
||||
const coins =
|
||||
await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 20);
|
||||
|
||||
for (const coin of coins) {
|
||||
const meta = await txindexer.getMeta(coin.hash);
|
||||
assert.bufferEqual(meta.tx.hash(), coin.hash);
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle indexing a reorg', async () => {
|
||||
await reorg(chain, cpu, 10);
|
||||
|
||||
assert.strictEqual(txindexer.state.startHeight, 31);
|
||||
assert.strictEqual(addrindexer.state.startHeight, 31);
|
||||
|
||||
const coins =
|
||||
await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 31);
|
||||
|
||||
for (const coin of coins) {
|
||||
const meta = await txindexer.getMeta(coin.hash);
|
||||
assert.bufferEqual(meta.tx.hash(), coin.hash);
|
||||
}
|
||||
});
|
||||
});
|
||||
@ -28,6 +28,8 @@ const node = new FullNode({
|
||||
network: 'regtest',
|
||||
workers: true,
|
||||
plugins: [require('../lib/wallet/plugin')],
|
||||
indexTX: true,
|
||||
indexAddress: true,
|
||||
port: ports.p2p,
|
||||
httpPort: ports.node,
|
||||
env: {
|
||||
@ -756,6 +758,55 @@ describe('Node', function() {
|
||||
assert.strictEqual(tx1.txid(), tx2.txid());
|
||||
});
|
||||
|
||||
it('should get tx by hash', async () => {
|
||||
const block = await mineBlock();
|
||||
await chain.add(block);
|
||||
|
||||
const tx = block.txs[0];
|
||||
const hash = tx.hash();
|
||||
const hasTX = await node.hasTX(hash);
|
||||
|
||||
assert.strictEqual(hasTX, true);
|
||||
|
||||
const tx2 = await node.getTX(hash);
|
||||
assert.strictEqual(tx.txid(), tx2.txid());
|
||||
|
||||
const meta = await node.getMeta(hash);
|
||||
assert.strictEqual(meta.tx.txid(), tx2.txid());
|
||||
});
|
||||
|
||||
it('should get coin/tx by addr', async () => {
|
||||
const addr = await wallet.receiveAddress();
|
||||
const mtx = await wallet.createTX({
|
||||
rate: 100000,
|
||||
outputs: [{
|
||||
value: 100000,
|
||||
address: addr
|
||||
}]
|
||||
});
|
||||
|
||||
await wallet.sign(mtx);
|
||||
|
||||
const tx = mtx.toTX();
|
||||
const job = await miner.createJob();
|
||||
|
||||
job.addTX(tx, mtx.view);
|
||||
job.refresh();
|
||||
|
||||
const block = await job.mineAsync();
|
||||
await chain.add(block);
|
||||
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
|
||||
const txs = await node.getTXByAddress(addr.hash);
|
||||
const tx2 = txs[0];
|
||||
assert.strictEqual(tx.txid(), tx2.txid());
|
||||
|
||||
const coins = await node.getCoinsByAddress(addr.hash);
|
||||
const coin = coins[0];
|
||||
assert.strictEqual(tx.txid(), coin.txid());
|
||||
});
|
||||
|
||||
it('should cleanup', async () => {
|
||||
consensus.COINBASE_MATURITY = 100;
|
||||
await node.close();
|
||||
|
||||
63
test/util/reorg.js
Normal file
63
test/util/reorg.js
Normal file
@ -0,0 +1,63 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('./assert');
|
||||
const Chain = require('../../lib/blockchain/chain');
|
||||
const CPUMiner = require('../../lib/mining/cpuminer');
|
||||
|
||||
/**
|
||||
* Reorgs the chain to given height using miners.
|
||||
* @param {Chain} chain chain
|
||||
* @param {CPUMiner} cpu cpuminer
|
||||
* @param {Number} height height
|
||||
* @returns {Promise} null
|
||||
*/
|
||||
async function reorg(chain, cpu, height) {
|
||||
assert(chain instanceof Chain);
|
||||
assert(cpu instanceof CPUMiner);
|
||||
assert(typeof height === 'number');
|
||||
|
||||
let tip1, tip2 = null;
|
||||
for (let i = 0; i < height; i++) {
|
||||
const job1 = await cpu.createJob(tip1);
|
||||
const job2 = await cpu.createJob(tip2);
|
||||
|
||||
const blk1 = await job1.mineAsync();
|
||||
const blk2 = await job2.mineAsync();
|
||||
|
||||
const hash1 = blk1.hash();
|
||||
const hash2 = blk2.hash();
|
||||
|
||||
assert(await chain.add(blk1));
|
||||
assert(await chain.add(blk2));
|
||||
|
||||
assert.bufferEqual(chain.tip.hash, hash1);
|
||||
|
||||
tip1 = await chain.getEntry(hash1);
|
||||
tip2 = await chain.getEntry(hash2);
|
||||
|
||||
assert(tip1);
|
||||
assert(tip2);
|
||||
|
||||
assert(!await chain.isMainChain(tip2));
|
||||
}
|
||||
|
||||
const entry = await chain.getEntry(tip2.hash);
|
||||
assert(entry);
|
||||
assert.strictEqual(chain.height, entry.height);
|
||||
|
||||
const block = await cpu.mineBlock(entry);
|
||||
assert(block);
|
||||
|
||||
let forked = false;
|
||||
chain.once('reorganize', () => {
|
||||
forked = true;
|
||||
});
|
||||
|
||||
assert(await chain.add(block));
|
||||
|
||||
assert(forked);
|
||||
assert.bufferEqual(chain.tip.hash, block.hash());
|
||||
assert(chain.tip.chainwork.gt(tip1.chainwork));
|
||||
}
|
||||
|
||||
module.exports = reorg;
|
||||
Loading…
Reference in New Issue
Block a user