indexer: add module indexer

module indexer introduces a extensible architecture for indexing the
chain. It provides a base class which handles syncing with the chain,
handling re-orgs, interruptions, dynamic toggling, etc. TXIndexer
and AddrIndexer are provided for indexing transactions and addresses,
using the same flags as before i.e --index-tx and --index-address.
Indexes are stored in a different database and can be maintained
independently of the chain.

lib/bcoin-browser.js

@@ -55,6 +55,13 @@ bcoin.HDPrivateKey = require('./hd/private');
 bcoin.HDPublicKey = require('./hd/public');
 bcoin.Mnemonic = require('./hd/mnemonic');
 
+// Index
+bcoin.indexer = require('./indexer');
+bcoin.Indexer = require('./indexer/indexer');
+bcoin.ChainClient = require('./indexer/chainclient');
+bcoin.TXIndexer = require('./indexer/txindexer');
+bcoin.AddrIndexer = require('./indexer/addrindexer');
+
 // Mempool
 bcoin.mempool = require('./mempool');
 bcoin.Fees = require('./mempool/fees');

lib/bcoin.js

@@ -76,6 +76,13 @@ bcoin.define('HDPrivateKey', './hd/private');
 bcoin.define('HDPublicKey', './hd/public');
 bcoin.define('Mnemonic', './hd/mnemonic');
 
+// Index
+bcoin.define('indexer', './indexer');
+bcoin.define('Indexer', './indexer/indexer');
+bcoin.define('ChainClient', './indexer/chainclient');
+bcoin.define('TXIndexer', './indexer/txindexer');
+bcoin.define('AddrIndexer', './indexer/addrindexer');
+
 // Mempool
 bcoin.define('mempool', './mempool');
 bcoin.define('Fees', './mempool/fees');

lib/blockchain/chain.js

@@ -2059,14 +2059,14 @@ class Chain extends AsyncEmitter {
 
   /**
    * Get coin viewpoint (spent).
-   * @param {TX} tx
+   * @param {TXMeta} meta
    * @returns {Promise} - Returns {@link CoinView}.
    */
 
-  async getSpentView(tx) {
+  async getSpentView(meta) {
     const unlock = await this.locker.lock();
     try {
-      return await this.db.getSpentView(tx);
+      return await this.db.getSpentView(meta);
     } finally {
       unlock();
     }
@@ -2766,11 +2766,6 @@ class ChainOptions {
       this.compression = options.compression;
     }
 
-    if (options.prune != null) {
-      assert(typeof options.prune === 'boolean');
-      this.prune = options.prune;
-    }
-
     if (options.indexTX != null) {
       assert(typeof options.indexTX === 'boolean');
       this.indexTX = options.indexTX;
@@ -2781,6 +2776,11 @@ class ChainOptions {
       this.indexAddress = options.indexAddress;
     }
 
+    if (options.prune != null) {
+      assert(typeof options.prune === 'boolean');
+      this.prune = options.prune;
+    }
+
     if (options.forceFlags != null) {
       assert(typeof options.forceFlags === 'boolean');
       this.forceFlags = options.forceFlags;

lib/blockchain/chaindb.js

@@ -11,7 +11,7 @@ const assert = require('bsert');
 const bdb = require('bdb');
 const bio = require('bufio');
 const LRU = require('blru');
-const {BufferMap, BufferSet} = require('buffer-map');
+const {BufferMap} = require('buffer-map');
 const Amount = require('../btc/amount');
 const Network = require('../protocol/network');
 const CoinView = require('../coins/coinview');
@@ -20,9 +20,7 @@ const layout = require('./layout');
 const consensus = require('../protocol/consensus');
 const Block = require('../primitives/block');
 const Outpoint = require('../primitives/outpoint');
-const Address = require('../primitives/address');
 const ChainEntry = require('./chainentry');
-const TXMeta = require('../primitives/txmeta');
 const CoinEntry = require('../coins/coinentry');
 
 /**
@@ -573,18 +571,12 @@ class ChainDB {
     if (!options.prune && flags.prune)
       throw new Error('Cannot retroactively unprune.');
 
-    if (options.indexTX && !flags.indexTX)
+    if (options.prune && options.indexTX && !flags.indexTX)
       throw new Error('Cannot retroactively enable TX indexing.');
 
-    if (!options.indexTX && flags.indexTX)
+    if (options.prune && options.indexAddress && !flags.indexAddress)
-      throw new Error('Cannot retroactively disable TX indexing.');
-
-    if (options.indexAddress && !flags.indexAddress)
       throw new Error('Cannot retroactively enable address indexing.');
 
-    if (!options.indexAddress && flags.indexAddress)
-      throw new Error('Cannot retroactively disable address indexing.');
-
     if (needsSave) {
       await this.logger.info('Rewriting chain flags.');
       await this.saveFlags();
@@ -978,30 +970,16 @@ class ChainDB {
 
   /**
    * Get coin viewpoint (historical).
-   * @param {TX} tx
+   * @param {TXMeta} meta
    * @returns {Promise} - Returns {@link CoinView}.
    */
 
-  async getSpentView(tx) {
+  async getSpentView(meta) {
-    const view = await this.getCoinView(tx);
+    process.emitWarning(
-
+      'deprecated, use node.txindex.getSpentView',
-    for (const {prevout} of tx.inputs) {
+      'DeprecationWarning'
-      if (view.hasEntry(prevout))
+    );
-        continue;
+    return null;
-
-      const {hash, index} = prevout;
-      const meta = await this.getMeta(hash);
-
-      if (!meta)
-        continue;
-
-      const {tx, height} = meta;
-
-      if (index < tx.outputs.length)
-        view.addIndex(tx, index, height);
-    }
-
-    return view;
   }
 
   /**
@@ -1083,152 +1061,105 @@ class ChainDB {
   /**
    * Get a transaction with metadata.
    * @param {Hash} hash
+   * @deprecated
    * @returns {Promise} - Returns {@link TXMeta}.
    */
 
   async getMeta(hash) {
-    if (!this.options.indexTX)
+    process.emitWarning(
-      return null;
+      'deprecated, use node.txindex.getMeta',
-
+      'DeprecationWarning'
-    const data = await this.db.get(layout.t.encode(hash));
+    );
-
+    return null;
-    if (!data)
-      return null;
-
-    return TXMeta.fromRaw(data);
   }
 
   /**
    * Retrieve a transaction.
    * @param {Hash} hash
+   * @deprecated
    * @returns {Promise} - Returns {@link TX}.
    */
 
   async getTX(hash) {
-    const meta = await this.getMeta(hash);
+    process.emitWarning(
-
+      'deprecated, use node.txindex.getTX',
-    if (!meta)
+      'DeprecationWarning'
-      return null;
+    );
-
+    return null;
-    return meta.tx;
   }
 
   /**
    * @param {Hash} hash
+   * @deprecated
    * @returns {Promise} - Returns Boolean.
    */
 
   async hasTX(hash) {
-    if (!this.options.indexTX)
+    process.emitWarning(
-      return false;
+      'deprecated, use node.txindex.hasTX',
-
+      'DeprecationWarning'
-    return this.db.has(layout.t.encode(hash));
+    );
+    return false;
   }
 
   /**
    * Get all coins pertinent to an address.
    * @param {Address[]} addrs
+   * @deprecated
    * @returns {Promise} - Returns {@link Coin}[].
    */
 
   async getCoinsByAddress(addrs) {
-    if (!this.options.indexAddress)
+    process.emitWarning(
-      return [];
+      'deprecated, use node.addrindex.getCoinsByAddress',
-
+      'DeprecationWarning'
-    if (!Array.isArray(addrs))
+    );
-      addrs = [addrs];
+    return [];
-
-    const coins = [];
-
-    for (const addr of addrs) {
-      const hash = Address.getHash(addr);
-
-      const keys = await this.db.keys({
-        gte: layout.C.min(hash),
-        lte: layout.C.max(hash),
-        parse: (key) => {
-          const [, txid, index] = layout.C.decode(key);
-          return [txid, index];
-        }
-      });
-
-      for (const [hash, index] of keys) {
-        const coin = await this.getCoin(hash, index);
-        assert(coin);
-        coins.push(coin);
-      }
-    }
-
-    return coins;
   }
 
   /**
    * Get all transaction hashes to an address.
    * @param {Address[]} addrs
+   * @deprecated
    * @returns {Promise} - Returns {@link Hash}[].
    */
 
   async getHashesByAddress(addrs) {
-    if (!this.options.indexTX || !this.options.indexAddress)
+    process.emitWarning(
-      return [];
+      'deprecated, use node.addrindex.getHashesByAddress',
-
+      'DeprecationWarning'
-    const set = new BufferSet();
+    );
-
+    return [];
-    for (const addr of addrs) {
-      const hash = Address.getHash(addr);
-
-      await this.db.keys({
-        gte: layout.T.min(hash),
-        lte: layout.T.max(hash),
-        parse: (key) => {
-          const [, txid] = layout.T.decode(key);
-          set.add(txid);
-        }
-      });
-    }
-
-    return set.toArray();
   }
 
   /**
    * Get all transactions pertinent to an address.
    * @param {Address[]} addrs
+   * @deprecated
    * @returns {Promise} - Returns {@link TX}[].
    */
 
   async getTXByAddress(addrs) {
-    const mtxs = await this.getMetaByAddress(addrs);
+    process.emitWarning(
-    const out = [];
+      'deprecated, use node.addrindex.getHashesByAddress',
-
+      'DeprecationWarning'
-    for (const mtx of mtxs)
+    );
-      out.push(mtx.tx);
+    return [];
-
-    return out;
   }
 
   /**
    * Get all transactions pertinent to an address.
    * @param {Address[]} addrs
+   * @deprecated
    * @returns {Promise} - Returns {@link TXMeta}[].
    */
 
   async getMetaByAddress(addrs) {
-    if (!this.options.indexTX || !this.options.indexAddress)
+    process.emitWarning(
-      return [];
+      'deprecated, use node.addrindex.getMetaByAddress',
-
+      'DeprecationWarning'
-    if (!Array.isArray(addrs))
+    );
-      addrs = [addrs];
+    return [];
-
-    const hashes = await this.getHashesByAddress(addrs);
-    const mtxs = [];
-
-    for (const hash of hashes) {
-      const mtx = await this.getMeta(hash);
-      assert(mtx);
-      mtxs.push(mtx);
-    }
-
-    return mtxs;
   }
 
   /**
@@ -1771,9 +1702,6 @@ class ChainDB {
 
         this.pending.add(output);
       }
-
-      // Index the transaction if enabled.
-      this.indexTX(tx, view, entry, i);
     }
 
     // Commit new coin state.
@@ -1828,9 +1756,6 @@ class ChainDB {
 
         this.pending.spend(output);
       }
-
-      // Remove from transaction index.
-      this.unindexTX(tx, view);
     }
 
     // Undo coins should be empty.
@@ -1882,105 +1807,6 @@ class ChainDB {
     b.put(layout.O.encode(), flags.toRaw());
     return b.write();
   }
-
-  /**
-   * Index a transaction by txid and address.
-   * @private
-   * @param {TX} tx
-   * @param {CoinView} view
-   * @param {ChainEntry} entry
-   * @param {Number} index
-   */
-
-  indexTX(tx, view, entry, index) {
-    const hash = tx.hash();
-
-    if (this.options.indexTX) {
-      const meta = TXMeta.fromTX(tx, entry, index);
-
-      this.put(layout.t.encode(hash), meta.toRaw());
-
-      if (this.options.indexAddress) {
-        for (const addr of tx.getHashes(view))
-          this.put(layout.T.encode(addr, hash), null);
-      }
-    }
-
-    if (!this.options.indexAddress)
-      return;
-
-    if (!tx.isCoinbase()) {
-      for (const {prevout} of tx.inputs) {
-        const {hash, index} = prevout;
-        const coin = view.getOutput(prevout);
-        assert(coin);
-
-        const addr = coin.getHash();
-
-        if (!addr)
-          continue;
-
-        this.del(layout.C.encode(addr, hash, index));
-      }
-    }
-
-    for (let i = 0; i < tx.outputs.length; i++) {
-      const output = tx.outputs[i];
-      const addr = output.getHash();
-
-      if (!addr)
-        continue;
-
-      this.put(layout.C.encode(addr, hash, i), null);
-    }
-  }
-
-  /**
-   * Remove transaction from index.
-   * @private
-   * @param {TX} tx
-   * @param {CoinView} view
-   */
-
-  unindexTX(tx, view) {
-    const hash = tx.hash();
-
-    if (this.options.indexTX) {
-      this.del(layout.t.encode(hash));
-      if (this.options.indexAddress) {
-        for (const addr of tx.getHashes(view))
-          this.del(layout.T.encode(addr, hash));
-      }
-    }
-
-    if (!this.options.indexAddress)
-      return;
-
-    if (!tx.isCoinbase()) {
-      for (const {prevout} of tx.inputs) {
-        const {hash, index} = prevout;
-        const coin = view.getOutput(prevout);
-        assert(coin);
-
-        const addr = coin.getHash();
-
-        if (!addr)
-          continue;
-
-        this.put(layout.C.encode(addr, hash, index), null);
-      }
-    }
-
-    for (let i = 0; i < tx.outputs.length; i++) {
-      const output = tx.outputs[i];
-      const addr = output.getHash();
-
-      if (!addr)
-        continue;
-
-      this.del(layout.C.encode(addr, hash, i));
-    }
-  }
 }
 
 /**

lib/blockchain/layout.js

@@ -20,12 +20,12 @@ const bdb = require('bdb');
  *   n[hash] -> next hash
  *   p[hash] -> tip index
  *   b[hash] -> block (deprecated)
- *   t[hash] -> extended tx
+ *   t[hash] -> extended tx (deprecated)
  *   c[hash] -> coins
  *   u[hash] -> undo coins (deprecated)
  *   v[bit][hash] -> versionbits state
- *   T[addr-hash][hash] -> dummy (tx by address)
+ *   T[addr-hash][hash] -> dummy (tx by address) (deprecated)
- *   C[addr-hash][hash][index] -> dummy (coin by address)
+ *   C[addr-hash][hash][index] -> dummy (coin by address) (deprecated)
  */
 
 const layout = {

lib/indexer/addrindexer.js

@@ -0,0 +1,197 @@
+/*!
+ * addrindexer.js - addr indexer
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const bdb = require('bdb');
+const {BufferSet} = require('buffer-map');
+const layout = require('./layout');
+const Address = require('../primitives/address');
+const Indexer = require('./indexer');
+
+/*
+ * AddrIndexer Database Layout:
+ *  T[addr-hash][hash] -> dummy (tx by address)
+ *  C[addr-hash][hash][index] -> dummy (coin by address)
+*/
+
+Object.assign(layout, {
+  T: bdb.key('T', ['hash', 'hash256']),
+  C: bdb.key('C', ['hash', 'hash256', 'uint32'])
+});
+
+/**
+ * AddrIndexer
+ * @alias module:indexer.AddrIndexer
+ * @extends Indexer
+ */
+
+class AddrIndexer extends Indexer {
+  /**
+   * Create a indexer
+   * @constructor
+   * @param {Object} options
+   */
+
+  constructor(options) {
+    super('addr', options);
+
+    this.db = bdb.create(this.options);
+  }
+
+  /**
+   * Index transactions by address.
+   * @private
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @param {CoinView} view
+   */
+
+  async indexBlock(entry, block, view) {
+    const b = this.db.batch();
+
+    for (let i = 0; i < block.txs.length; i++) {
+      const tx = block.txs[i];
+      const hash = tx.hash();
+      for (const addr of tx.getHashes(view))
+        b.put(layout.T.encode(addr, hash), null);
+
+      if (!tx.isCoinbase()) {
+        for (const {prevout} of tx.inputs) {
+          const {hash, index} = prevout;
+          const coin = view.getOutput(prevout);
+          assert(coin);
+
+          const addr = coin.getHash();
+
+          if (!addr)
+            continue;
+
+          b.del(layout.C.encode(addr, hash, index));
+        }
+      }
+
+      for (let i = 0; i < tx.outputs.length; i++) {
+        const output = tx.outputs[i];
+        const addr = output.getHash();
+
+        if (!addr)
+          continue;
+
+        b.put(layout.C.encode(addr, hash, i), null);
+      }
+    }
+
+    return b.write();
+  }
+
+  /**
+   * Remove addresses from index.
+   * @private
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @param {CoinView} view
+   */
+
+  async unindexBlock(entry, block, view) {
+    const b = this.db.batch();
+    for (let i = 0; i < block.txs.length; i++) {
+      const tx = block.txs[i];
+      const hash = tx.hash();
+      for (const addr of tx.getHashes(view))
+        b.del(layout.T.encode(addr, hash));
+
+      if (!tx.isCoinbase()) {
+        for (const {prevout} of tx.inputs) {
+          const {hash, index} = prevout;
+          const coin = view.getOutput(prevout);
+          assert(coin);
+
+          const addr = coin.getHash();
+
+          if (!addr)
+            continue;
+
+          b.put(layout.C.encode(addr, hash, index), null);
+        }
+      }
+
+      for (let i = 0; i < tx.outputs.length; i++) {
+        const output = tx.outputs[i];
+        const addr = output.getHash();
+
+        if (!addr)
+          continue;
+
+        b.del(layout.C.encode(addr, hash, i));
+      }
+    }
+
+    return b.write();
+  }
+
+  /**
+   * Get all coins pertinent to an address.
+   * @param {Address[]} addrs
+   * @returns {Promise} - Returns {@link Coin}[].
+   */
+
+  async getCoinsByAddress(addrs) {
+    if (!Array.isArray(addrs))
+      addrs = [addrs];
+
+    const coins = [];
+
+    for (const addr of addrs) {
+      const hash = Address.getHash(addr);
+
+      const keys = await this.db.keys({
+        gte: layout.C.min(hash),
+        lte: layout.C.max(hash),
+        parse: (key) => {
+          const [, txid, index] = layout.C.decode(key);
+          return [txid, index];
+        }
+      });
+
+      for (const [hash, index] of keys) {
+        const coin = await this.client.getCoin(hash, index);
+        assert(coin);
+        coins.push(coin);
+      }
+    }
+
+    return coins;
+  }
+
+  /**
+   * Get all transaction hashes to an address.
+   * @param {Address[]} addrs
+   * @returns {Promise} - Returns {@link Hash}[].
+   */
+
+  async getHashesByAddress(addrs) {
+    const set = new BufferSet();
+
+    for (const addr of addrs) {
+      const hash = Address.getHash(addr);
+
+      await this.db.keys({
+        gte: layout.T.min(hash),
+        lte: layout.T.max(hash),
+        parse: (key) => {
+          const [, txid] = layout.T.decode(key);
+          set.add(txid);
+        }
+      });
+    }
+
+    return set.toArray();
+  }
+}
+
+module.exports = AddrIndexer;

lib/indexer/chainclient.js

@@ -0,0 +1,200 @@
+/*!
+ * chainclient.js - chain client for bcoin
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const AsyncEmitter = require('bevent');
+const Chain = require('../blockchain/chain');
+
+/**
+ * Chain Client
+ * @extends AsyncEmitter
+ * @alias module:indexer.ChainClient
+ */
+
+class ChainClient extends AsyncEmitter {
+  /**
+   * Create a chain client.
+   * @constructor
+   * @param {Chain} chain
+   */
+
+  constructor(chain) {
+    super();
+
+    assert(chain instanceof Chain);
+
+    this.chain = chain;
+    this.network = chain.network;
+    this.opened = false;
+
+    this.init();
+  }
+
+  /**
+   * Initialize the client.
+   */
+
+  init() {
+    this.chain.on('connect', async (entry, block, view) => {
+      if (!this.opened)
+        return;
+
+      await this.emitAsync('block connect', entry, block, view);
+    });
+
+    this.chain.on('disconnect', async (entry, block, view) => {
+      if (!this.opened)
+        return;
+
+      await this.emitAsync('block disconnect', entry, block, view);
+    });
+
+    this.chain.on('reset', async (tip) => {
+      if (!this.opened)
+        return;
+
+      await this.emitAsync('chain reset', tip);
+    });
+  }
+
+  /**
+   * Open the client.
+   * @returns {Promise}
+   */
+
+  async open(options) {
+    assert(!this.opened, 'ChainClient is already open.');
+    this.opened = true;
+    setImmediate(() => this.emit('connect'));
+  }
+
+  /**
+   * Close the client.
+   * @returns {Promise}
+   */
+
+  async close() {
+    assert(this.opened, 'ChainClient is not open.');
+    this.opened = false;
+    setImmediate(() => this.emit('disconnect'));
+  }
+
+  /**
+   * Get chain tip.
+   * @returns {Promise}
+   */
+
+  async getTip() {
+    return this.chain.tip;
+  }
+
+  /**
+   * Get chain entry.
+   * @param {Hash} hash
+   * @returns {Promise} - Returns {@link ChainEntry}.
+   */
+
+  async getEntry(hash) {
+    const entry = await this.chain.getEntry(hash);
+
+    if (!entry)
+      return null;
+
+    if (!await this.chain.isMainChain(entry))
+      return null;
+
+    return entry;
+  }
+
+  /**
+   * Get a coin (unspents only).
+   * @param {Hash} hash
+   * @param {Number} index
+   * @returns {Promise} - Returns {@link Coin}.
+   */
+
+  async getCoin(hash, index) {
+    return this.chain.getCoin(hash, index);
+  }
+
+  /**
+   * Get hash range.
+   * @param {Number} start
+   * @param {Number} end
+   * @returns {Promise}
+   */
+
+  async getHashes(start = -1, end = -1) {
+    return this.chain.getHashes(start, end);
+  }
+
+  /**
+   * Get block
+   * @param {Hash} hash
+   * @returns {Promise} - Returns {@link Block}
+   */
+
+  async getBlock(hash) {
+    const block = await this.chain.getBlock(hash);
+
+    if (!block)
+      return null;
+
+    return block;
+  }
+
+  /**
+   * Get a historical block coin viewpoint.
+   * @param {Block} hash
+   * @returns {Promise} - Returns {@link CoinView}.
+   */
+
+  async getBlockView(block) {
+    return this.chain.getBlockView(block);
+  }
+
+  /**
+   * Get coin viewpoint.
+   * @param {TX} tx
+   * @returns {Promise} - Returns {@link CoinView}.
+   */
+
+  async getCoinView(tx) {
+    return this.chain.getCoinView(tx);
+  }
+
+  /**
+   * Rescan for any missed blocks.
+   * @param {Number} start - Start block.
+   * @returns {Promise}
+   */
+
+  async rescan(start) {
+    for (let i = start; ; i++) {
+      const entry = await this.getEntry(i);
+      if (!entry) {
+        await this.emitAsync('chain tip');
+        break;
+      };
+
+      const block = await this.getBlock(entry.hash);
+      assert(block);
+
+      const view = await this.getBlockView(block);
+      assert(view);
+
+      await this.emitAsync('block rescan', entry, block, view);
+    }
+  };
+}
+
+/*
+ * Expose
+ */
+
+module.exports = ChainClient;

lib/indexer/index.js

@@ -0,0 +1,16 @@
+/*!
+ * index.js - indexer for bcoin
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+/**
+ * @module indexer
+ */
+
+exports.Indexer = require('./indexer');
+exports.TXIndexer = require('./txindexer');
+exports.AddrIndexer = require('./addrindexer');
+exports.ChainClient = require('./chainclient');

lib/indexer/indexer.js

@@ -0,0 +1,812 @@
+/*!
+ * indexer.js - storage for indexes
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const path = require('path');
+const fs = require('bfile');
+const EventEmitter = require('events');
+const {Lock} = require('bmutex');
+const Logger = require('blgr');
+const Network = require('../protocol/network');
+const layout = require('./layout');
+const records = require('./records');
+const ChainClient = require('./chainclient');
+const NullClient = require('./nullclient');
+
+const {
+  ChainState,
+  BlockMeta
+} = records;
+
+/**
+ * Indexer
+ * @alias module:indexer.Indexer
+ * @extends EventEmitter
+ * @property {IndexerDB} db
+ * @property {Number} height
+ * @property {ChainState} state
+ * @emits Indexer#chain tip
+ */
+
+class Indexer extends EventEmitter {
+  /**
+   * Create a index db.
+   * @constructor
+   * @param {String} module
+   * @param {Object} options
+   */
+
+  constructor(module, options) {
+    super();
+
+    assert(typeof module === 'string');
+    assert(module.length > 0);
+
+    this.options = new IndexOptions(module, options);
+
+    this.network = this.options.network;
+    this.logger = this.options.logger.context(`${module}indexer`);
+    this.client = this.options.client || new NullClient(this);
+    this.db = null;
+    this.rescanning = false;
+
+    this.state = new ChainState();
+    this.height = 0;
+
+    this.lock = new Lock();
+
+    this.init();
+  }
+
+  /**
+   * Initialize indexdb.
+   * @private
+   */
+
+  init() {
+    this._bind();
+  }
+
+  /**
+   * Bind to chain events.
+   * @private
+   */
+
+  _bind() {
+    this.client.on('error', (err) => {
+      this.emit('error', err);
+    });
+
+    this.client.on('connect', async () => {
+      try {
+        await this.syncNode();
+      } catch (e) {
+        this.emit('error', e);
+      }
+    });
+
+    this.client.on('block connect', async (entry, block, view) => {
+      if (this.rescanning)
+        return;
+      try {
+        await this.addBlock(entry, block, view);
+      } catch (e) {
+        this.emit('error', e);
+      }
+    });
+
+    this.client.on('block disconnect', async (entry, block, view) => {
+      if (this.rescanning)
+        return;
+      try {
+        await this.removeBlock(entry, block, view);
+      } catch (e) {
+        this.emit('error', e);
+      }
+    });
+
+    this.client.on('block rescan', async (entry, block, view) => {
+      try {
+        await this.rescanBlock(entry, block, view);
+      } catch (e) {
+        this.emit('error', e);
+      }
+    });
+
+    this.client.on('chain reset', async (tip) => {
+      try {
+        await this.resetChain(tip);
+      } catch (e) {
+        this.emit('error', e);
+      }
+    });
+
+    this.client.on('chain tip', async () => {
+      this.logger.debug('Indexer: finished rescan');
+      const tip = await this.getTip();
+      this.emit('chain tip', tip);
+    });
+  }
+
+  /**
+   * Ensure prefix directory (prefix/index).
+   * @returns {Promise}
+   */
+
+  async ensure() {
+    if (fs.unsupported)
+      return undefined;
+
+    if (this.options.memory)
+      return undefined;
+
+    return fs.mkdirp(this.options.prefix);
+  }
+
+  /**
+   * Open the indexdb, wait for the database to load.
+   * @returns {Promise}
+   */
+
+  async open() {
+    await this.ensure();
+    await this.db.open();
+    await this.db.verify(layout.V.encode(), 'index', 0);
+
+    await this.verifyNetwork();
+
+    await this.connect();
+  }
+
+  /**
+   * Verify network.
+   * @returns {Promise}
+   */
+
+  async verifyNetwork() {
+    const raw = await this.db.get(layout.O.encode());
+
+    if (!raw) {
+      const b = this.db.batch();
+      b.put(layout.O.encode(), fromU32(this.network.magic));
+      return b.write();
+    }
+
+    const magic = raw.readUInt32LE(0, true);
+
+    if (magic !== this.network.magic)
+      throw new Error('Network mismatch for Indexer.');
+
+    return undefined;
+  }
+
+  /**
+   * Close the indexdb, wait for the database to close.
+   * @returns {Promise}
+   */
+
+  async close() {
+    await this.disconnect();
+    return this.db.close();
+  }
+
+  /**
+   * Connect to the chain server (client required).
+   * @returns {Promise}
+   */
+
+  async connect() {
+    return this.client.open();
+  }
+
+  /**
+   * Disconnect from chain server (client required).
+   * @returns {Promise}
+   */
+
+  async disconnect() {
+    return this.client.close();
+  }
+
+  /**
+   * Sync state with server on every connect.
+   * @returns {Promise}
+   */
+
+  async syncNode() {
+    const unlock = await this.lock.lock();
+    try {
+      this.logger.info('Resyncing from server...');
+      await this.syncState();
+      await this.syncChain();
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * Initialize and write initial sync state.
+   * @returns {Promise}
+   */
+
+  async syncState() {
+    const cache = await this.getState();
+
+    if (cache) {
+      this.state = cache;
+      this.height = cache.height;
+
+      this.logger.info(
+        'Indexer loaded (height=%d, start=%d).',
+        this.state.height,
+        this.state.startHeight);
+      return undefined;
+    }
+
+    this.logger.info('Initializing database state from server.');
+
+    const b = this.db.batch();
+    const hashes = await this.client.getHashes();
+
+    let tip = null;
+
+    for (let height = 0; height < hashes.length; height++) {
+      const hash = hashes[height];
+      const meta = new BlockMeta(hash, height);
+      b.put(layout.h.encode(height), meta.toHash());
+      tip = meta;
+    }
+
+    assert(tip);
+
+    const state = this.state.clone();
+    state.startHeight = 0;
+    state.height = tip.height;
+
+    b.put(layout.R.encode(), state.toRaw());
+
+    await b.write();
+
+    this.state = state;
+    this.height = state.height;
+
+    return undefined;
+  }
+
+  /**
+   * Connect and sync with the chain server.
+   * @private
+   * @returns {Promise}
+   */
+
+  async syncChain() {
+    let height = this.state.height;
+
+    this.logger.info('Syncing state from height %d.', height);
+
+    // re-org when we're offline might
+    // leave chain in different state.
+    // scan chain backwards until we
+    // find a known 'good' height
+    for (;;) {
+      const tip = await this.getBlock(height);
+      assert(tip);
+
+      if (await this.client.getEntry(tip.hash))
+        break;
+
+      assert(height !== 0);
+      height -= 1;
+    }
+
+    // start scan from last indexed OR
+    // last known 'good' height whichever
+    // is lower, because `scan` scans from
+    // low to high blocks
+    if (this.state.startHeight < height)
+      height = this.state.startHeight;
+
+    this.logger.spam('Starting block rescan from: %d.', height);
+    return this.scan(height);
+  }
+
+  /**
+   * Rescan a block.
+   * @private
+   * @param {ChainEntry} entry
+   * @param {TX[]} txs
+   * @returns {Promise}
+   */
+
+  async rescanBlock(entry, block, view) {
+    this.logger.spam('Rescanning block: %d.', entry.height);
+
+    if (!this.rescanning) {
+      this.logger.warning('Unsolicited rescan block: %d.', entry.height);
+      return;
+    }
+
+    if (entry.height % 1000 === 0)
+      this.logger.debug('rescanned block: %d.', entry.height);
+
+    if (entry.height > this.state.height + 1) {
+      this.logger.warning('Rescan block too high: %d.', entry.height);
+      return;
+    }
+
+    try {
+      await this._addBlock(entry, block, view);
+    } catch (e) {
+      this.emit('error', e);
+      throw e;
+    }
+  }
+
+  /**
+   * Rescan blockchain from a given height.
+   * @private
+   * @param {Number?} height
+   * @returns {Promise}
+   */
+
+  async scan(height) {
+    assert((height >>> 0) === height, 'Indexer: Must pass in a height.');
+
+    await this.rollback(height);
+
+    const tip = this.state.height;
+
+    this.logger.info(
+      'Indexer is scanning %d blocks.',
+      tip - height + 1);
+
+    try {
+      this.rescanning = true;
+      this.logger.debug('rescanning from %d to %d', height, tip);
+      await this.client.rescan(height);
+    } finally {
+      this.rescanning = false;
+    }
+  }
+
+  /**
+   * Force a rescan.
+   * @param {Number} height
+   * @returns {Promise}
+   */
+
+  async rescan(height) {
+    const unlock = await this.lock.lock();
+    try {
+      return await this._rescan(height);
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * Force a rescan (without a lock).
+   * @private
+   * @param {Number} height
+   * @returns {Promise}
+   */
+
+  async _rescan(height) {
+    return this.scan(height);
+  }
+
+  /**
+   * Get the best block hash.
+   * @returns {Promise}
+   */
+
+  async getState() {
+    const data = await this.db.get(layout.R.encode());
+
+    if (!data)
+      return null;
+
+    return ChainState.fromRaw(data);
+  }
+
+  /**
+   * Sync the current chain state to tip.
+   * @param {BlockMeta} tip
+   * @returns {Promise}
+   */
+
+  async setTip(tip) {
+    const b = this.db.batch();
+    const state = this.state.clone();
+
+    if (tip.height < state.height) {
+      // Hashes ahead of our new tip
+      // that we need to delete.
+      while (state.height !== tip.height) {
+        b.del(layout.h.encode(state.height));
+        state.height -= 1;
+      }
+    } else if (tip.height > state.height) {
+      assert(tip.height === state.height + 1, 'Bad chain sync.');
+      state.height += 1;
+    }
+
+    state.startHeight = tip.height;
+
+    // Save tip and state.
+    b.put(layout.h.encode(tip.height), tip.toHash());
+    b.put(layout.R.encode(), state.toRaw());
+
+    await b.write();
+
+    this.state = state;
+    this.height = state.height;
+  }
+
+  /**
+   * Get a index block meta.
+   * @param {Hash} hash
+   * @returns {Promise}
+   */
+
+  async getBlock(height) {
+    const data = await this.db.get(layout.h.encode(height));
+
+    if (!data)
+      return null;
+
+    const block = new BlockMeta();
+    block.hash = data;
+    block.height = height;
+
+    return block;
+  }
+
+  /**
+   * Get index tip.
+   * @param {Hash} hash
+   * @returns {Promise}
+   */
+
+  async getTip() {
+    const tip = await this.getBlock(this.state.height);
+
+    if (!tip)
+      throw new Error('Indexer: Tip not found!');
+
+    return tip;
+  }
+
+  /**
+   * Sync with chain height.
+   * @param {Number} height
+   * @returns {Promise}
+   */
+
+  async rollback(height) {
+    if (height > this.state.height)
+      throw new Error('Indexer: Cannot rollback to the future.');
+
+    if (height === this.state.height) {
+      this.logger.info('Rolled back to same height (%d).', height);
+      return;
+    }
+
+    this.logger.info(
+      'Rolling back %d Indexer blocks to height %d.',
+      this.state.height - height, height);
+
+    const tip = await this.getBlock(height);
+    assert(tip);
+
+    await this.revert(tip.height);
+    await this.setTip(tip);
+  }
+
+  /**
+   * Add a block's transactions and write the new best hash.
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @returns {Promise}
+   */
+
+  async addBlock(entry, block, view) {
+    const unlock = await this.lock.lock();
+    try {
+      return await this._addBlock(entry, block, view);
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * Add a block's transactions without a lock.
+   * @private
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @returns {Promise}
+   */
+
+  async _addBlock(entry, block, view) {
+    const tip = BlockMeta.fromEntry(entry);
+
+    if (tip.height >= this.network.block.slowHeight && !this.rescanning)
+      this.logger.debug('Adding block: %d.', tip.height);
+
+    this.logger.spam('Adding block: %d.', entry.height);
+
+    if (tip.height === this.state.height) {
+      // We let blocks of the same height
+      // through specifically for rescans:
+      // we always want to rescan the last
+      // block since the state may have
+      // updated before the block was fully
+      // processed (in the case of a crash).
+      this.logger.warning('Already saw Indexer block (%d).', tip.height);
+    } else if (tip.height !== this.state.startHeight + 1) {
+      await this.scan(this.state.height);
+      return;
+    }
+
+    this.logger.spam('Indexing block: %d.', entry.height);
+
+    await this.indexBlock(entry, block, view);
+
+    // Sync the state to the new tip.
+    await this.setTip(tip);
+
+    return;
+  }
+
+  /**
+   * Process block indexing
+   * Indexers will implement this method to process the block for indexing
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @returns {Promise}
+   */
+
+  async indexBlock(entry, block, view) {
+    ;
+  }
+
+  /**
+   * Undo block indexing
+   * Indexers will implement this method to undo indexing for the block
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @returns {Promise}
+   */
+
+  async unindexBlock(entry, block, view) {
+    ;
+  }
+
+  /**
+   * Revert db to an older state.
+   * @param {Number} target
+   * @returns {Promise}
+   */
+
+  async revert(target) {
+    ;
+  }
+
+  /**
+   * Unconfirm a block's transactions
+   * and write the new best hash (SPV version).
+   * @param {ChainEntry} entry
+   * @returns {Promise}
+   */
+
+  async removeBlock(entry, block, view) {
+    const unlock = await this.lock.lock();
+    try {
+      return await this._removeBlock(entry, block, view);
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * Unconfirm a block's transactions.
+   * @private
+   * @param {ChainEntry} entry
+   * @returns {Promise}
+   */
+
+  async _removeBlock(entry, block, view) {
+    const tip = BlockMeta.fromEntry(entry);
+
+    this.logger.spam('Removing block: %d.', entry.height);
+
+    if (tip.height === 0)
+      throw new Error('Indexer: Bad disconnection (genesis block).');
+
+    if (tip.height > this.state.height) {
+      this.logger.warning(
+        'Indexer is disconnecting high blocks (%d).',
+        tip.height);
+      return;
+    }
+
+    if (tip.height !== this.state.height)
+      throw new Error('Indexer: Bad disconnection (height mismatch).');
+
+    this.logger.spam('Unindexing block: %d.', entry.height);
+
+    await this.unindexBlock(entry, block, view);
+
+    const prev = await this.getBlock(tip.height - 1);
+    assert(prev);
+
+    // Sync the state to the previous tip.
+    await this.setTip(prev);
+
+    return;
+  }
+
+  /**
+   * Handle a chain reset.
+   * @param {ChainEntry} entry
+   * @returns {Promise}
+   */
+
+  async resetChain(entry) {
+    const unlock = await this.lock.lock();
+    try {
+      return await this._resetChain(entry);
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * Handle a chain reset without a lock.
+   * @private
+   * @param {ChainEntry} entry
+   * @returns {Promise}
+   */
+
+  async _resetChain(entry) {
+    if (entry.height > this.state.height)
+      throw new Error('Indexer: Bad reset height.');
+
+    return this.rollback(entry.height);
+  }
+}
+
+/**
+ * Index Options
+ * @alias module:indexer.IndexOptions
+ */
+
+class IndexOptions {
+  /**
+   * Create index options.
+   * @constructor
+   * @param {String} module
+   * @param {Object} options
+   */
+
+  constructor(module, options) {
+    this.module = module;
+    this.network = Network.primary;
+    this.logger = Logger.global;
+    this.client = null;
+    this.chain = null;
+    this.indexers = null;
+
+    this.prefix = null;
+    this.location = null;
+    this.memory = true;
+    this.maxFiles = 64;
+    this.cacheSize = 16 << 20;
+    this.compression = true;
+
+    if (options)
+      this.fromOptions(options);
+  }
+
+  /**
+   * Inject properties from object.
+   * @private
+   * @param {Object} options
+   * @returns {IndexOptions}
+   */
+
+  fromOptions(options) {
+    if (options.network != null)
+      this.network = Network.get(options.network);
+
+    if (options.logger != null) {
+      assert(typeof options.logger === 'object');
+      this.logger = options.logger;
+    }
+
+    if (options.client != null) {
+      assert(typeof options.client === 'object');
+      this.client = options.client;
+    }
+
+    if (options.chain != null) {
+      assert(typeof options.chain === 'object');
+      this.client = new ChainClient(options.chain);
+    }
+
+    if (!this.client) {
+      throw new Error('Client is required');
+    }
+
+    if (options.prefix != null) {
+      assert(typeof options.prefix === 'string');
+      this.prefix = options.prefix;
+      this.prefix = path.join(this.prefix, 'index');
+      this.location = path.join(this.prefix, this.module);
+    }
+
+    if (options.location != null) {
+      assert(typeof options.location === 'string');
+      this.location = options.location;
+    }
+
+    if (options.memory != null) {
+      assert(typeof options.memory === 'boolean');
+      this.memory = options.memory;
+    }
+
+    if (options.maxFiles != null) {
+      assert((options.maxFiles >>> 0) === options.maxFiles);
+      this.maxFiles = options.maxFiles;
+    }
+
+    if (options.cacheSize != null) {
+      assert(Number.isSafeInteger(options.cacheSize) && options.cacheSize >= 0);
+      this.cacheSize = options.cacheSize;
+    }
+
+    if (options.compression != null) {
+      assert(typeof options.compression === 'boolean');
+      this.compression = options.compression;
+    }
+
+    return this;
+  }
+
+  /**
+   * Instantiate chain options from object.
+   * @param {Object} options
+   * @returns {IndexOptions}
+   */
+
+  static fromOptions(options) {
+    return new this().fromOptions(options);
+  }
+}
+
+/*
+ * Helpers
+ */
+
+/**
+ * fromU32
+ * read a 4 byte Uint32LE
+ * @param {Number} num number
+ * @returns {Buffer} buffer
+ */
+function fromU32(num) {
+  const data = Buffer.allocUnsafe(4);
+  data.writeUInt32LE(num, 0, true);
+  return data;
+}
+
+/*
+ * Expose
+ */
+
+module.exports = Indexer;

lib/indexer/layout.js

@@ -0,0 +1,31 @@
+/*!
+ * layout.js - indexer layout for bcoin
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const bdb = require('bdb');
+
+/*
+ * Index Database Layout:
+ * To be extended by indexer implementations
+ *  V -> db version
+ *  O -> flags
+ *  h[height] -> recent block hash
+ *  R -> chain sync state
+ */
+
+const layout = {
+  V: bdb.key('V'),
+  O: bdb.key('O'),
+  h: bdb.key('h', ['uint32']),
+  R: bdb.key('R')
+};
+
+/*
+ * Expose
+ */
+
+module.exports = layout;

lib/indexer/nullclient.js

@@ -0,0 +1,142 @@
+/*!
+ * nullclient.js - chain client for bcoin
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const assert = require('assert');
+const EventEmitter = require('events');
+
+/**
+ * Null Client
+ * Sort of a fake local client for separation of concerns.
+ * @alias module:indexer.NullClient
+ */
+
+class NullClient extends EventEmitter {
+  /**
+   * Create a client.
+   * @constructor
+   * @param {Chain} chain
+   */
+
+  constructor(chain) {
+    super();
+
+    this.chain = chain;
+    this.network = chain.network;
+    this.opened = false;
+  }
+
+  /**
+   * Open the client.
+   * @returns {Promise}
+   */
+
+  async open(options) {
+    assert(!this.opened, 'NullClient is already open.');
+    this.opened = true;
+    setImmediate(() => this.emit('connect'));
+  }
+
+  /**
+   * Close the client.
+   * @returns {Promise}
+   */
+
+  async close() {
+    assert(this.opened, 'NullClient is not open.');
+    this.opened = false;
+    setImmediate(() => this.emit('disconnect'));
+  }
+
+  /**
+   * Get chain tip.
+   * @returns {Promise}
+   */
+
+  async getTip() {
+    const {hash, height, time} = this.network.genesis;
+    return { hash, height, time };
+  }
+
+  /**
+   * Get chain entry.
+   * @param {Hash} hash
+   * @returns {Promise} - Returns {@link ChainEntry}.
+   */
+
+  async getEntry(hash) {
+    return { hash, height: 0, time: 0 };
+  }
+
+  /**
+   * Get a coin (unspents only).
+   * @param {Hash} hash
+   * @param {Number} index
+   * @returns {Promise} - Returns {@link Coin}.
+   */
+
+  async getCoin(hash, index) {
+    return null;
+  }
+
+  /**
+   * Get hash range.
+   * @param {Number} start
+   * @param {Number} end
+   * @returns {Promise}
+   */
+
+  async getHashes(start = -1, end = -1) {
+    return [this.network.genesis.hash];
+  }
+
+  /**
+   * Get block
+   * @param {Hash} hash
+   * @returns {Promise}
+   */
+
+  async getBlock(hash) {
+    return null;
+  }
+
+  /**
+   * Get a historical block coin viewpoint.
+   * @param {Block} hash
+   * @returns {Promise} - Returns {@link CoinView}.
+   */
+
+  async getBlockView(block) {
+    return null;
+  }
+
+  /**
+   * Get coin viewpoint.
+   * @param {TX} tx
+   * @returns {Promise} - Returns {@link CoinView}.
+   */
+
+  async getCoinView(tx) {
+    return null;
+  }
+
+  /**
+   * Rescan for any missed blocks.
+   * @param {Number} start - Start block.
+   * @returns {Promise}
+   */
+
+  async rescan(start) {
+    ;
+  }
+}
+
+/*
+ * Expose
+ */
+
+module.exports = NullClient;

lib/indexer/records.js

@@ -0,0 +1,221 @@
+/*!
+ * records.js - indexer records
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+/**
+ * @module lib/records
+ */
+
+const bio = require('bufio');
+const util = require('../utils/util');
+const consensus = require('../protocol/consensus');
+
+/**
+ * Chain State
+ * @alias module:indexer.ChainState
+ */
+
+class ChainState {
+  /**
+   * Create a chain state.
+   * @constructor
+   */
+
+  constructor() {
+    this.startHeight = 0;
+    this.height = 0;
+  }
+
+  /**
+   * Clone the state.
+   * @returns {ChainState}
+   */
+
+  clone() {
+    const state = new ChainState();
+    state.startHeight = this.startHeight;
+    state.height = this.height;
+    return state;
+  }
+
+  /**
+   * Inject properties from serialized data.
+   * @private
+   * @param {Buffer} data
+   */
+
+  fromRaw(data) {
+    const br = bio.read(data);
+
+    this.startHeight = br.readU32();
+    this.height = br.readU32();
+
+    return this;
+  }
+
+  /**
+   * Instantiate chain state from serialized data.
+   * @param {Buffer} data
+   * @returns {ChainState}
+   */
+
+  static fromRaw(data) {
+    return new this().fromRaw(data);
+  }
+
+  /**
+   * Serialize the chain state.
+   * @returns {Buffer}
+   */
+
+  toRaw() {
+    const bw = bio.write(8);
+
+    bw.writeU32(this.startHeight);
+    bw.writeU32(this.height);
+
+    return bw.render();
+  }
+}
+
+/**
+ * Block Meta
+ * @alias module:indexer.BlockMeta
+ */
+
+class BlockMeta {
+  /**
+   * Create block meta.
+   * @constructor
+   * @param {Hash} hash
+   * @param {Number} height
+   */
+
+  constructor(hash, height) {
+    this.hash = hash || consensus.NULL_HASH;
+    this.height = height != null ? height : -1;
+  }
+
+  /**
+   * Clone the block.
+   * @returns {BlockMeta}
+   */
+
+  clone() {
+    return new this.constructor(this.hash, this.height);
+  }
+
+  /**
+   * Get block meta hash as a buffer.
+   * @returns {Buffer}
+   */
+
+  toHash() {
+    return Buffer.from(this.hash, 'hex');
+  }
+
+  /**
+   * Instantiate block meta from chain entry.
+   * @private
+   * @param {IndexEntry} entry
+   */
+
+  fromEntry(entry) {
+    this.hash = entry.hash;
+    this.height = entry.height;
+    return this;
+  }
+
+  /**
+   * Instantiate block meta from json object.
+   * @private
+   * @param {Object} json
+   */
+
+  fromJSON(json) {
+    this.hash = util.revHex(json.hash);
+    this.height = json.height;
+    return this;
+  }
+
+  /**
+   * Instantiate block meta from serialized tip data.
+   * @private
+   * @param {Buffer} data
+   */
+
+  fromRaw(data) {
+    const br = bio.read(data);
+    this.hash = br.readHash('hex');
+    this.height = br.readI32();
+    return this;
+  }
+
+  /**
+   * Instantiate block meta from chain entry.
+   * @param {IndexEntry} entry
+   * @returns {BlockMeta}
+   */
+
+  static fromEntry(entry) {
+    return new this().fromEntry(entry);
+  }
+
+  /**
+   * Instantiate block meta from json object.
+   * @param {Object} json
+   * @returns {BlockMeta}
+   */
+
+  static fromJSON(json) {
+    return new this().fromJSON(json);
+  }
+
+  /**
+   * Instantiate block meta from serialized data.
+   * @param {Hash} hash
+   * @param {Buffer} data
+   * @returns {BlockMeta}
+   */
+
+  static fromRaw(data) {
+    return new this().fromRaw(data);
+  }
+
+  /**
+   * Serialize the block meta.
+   * @returns {Buffer}
+   */
+
+  toRaw() {
+    const bw = bio.write(36);
+    bw.writeHash(this.hash);
+    bw.writeI32(this.height);
+    return bw.render();
+  }
+
+  /**
+   * Convert the block meta to a more json-friendly object.
+   * @returns {Object}
+   */
+
+  toJSON() {
+    return {
+      hash: util.revHex(this.hash),
+      height: this.height
+    };
+  }
+}
+
+/*
+ * Expose
+ */
+
+exports.ChainState = ChainState;
+exports.BlockMeta = BlockMeta;
+
+module.exports = exports;

lib/indexer/txindexer.js

@@ -0,0 +1,151 @@
+/*!
+ * txindexer.js - tx indexer
+ * Copyright (c) 2018, the bcoin developers (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+const bdb = require('bdb');
+const layout = require('./layout');
+const TXMeta = require('../primitives/txmeta');
+const Indexer = require('./indexer');
+
+/*
+ * TXIndexer Database Layout:
+ *  t[hash] -> extended tx
+*/
+
+Object.assign(layout, {
+  t: bdb.key('t', ['hash256'])
+});
+
+/**
+ * TXIndexer
+ * @alias module:indexer.TXIndexer
+ * @extends Indexer
+ */
+
+class TXIndexer extends Indexer {
+  /**
+   * Create a indexer
+   * @constructor
+   * @param {Object} options
+   */
+
+  constructor(options) {
+    super('tx', options);
+
+    this.db = bdb.create(this.options);
+  }
+
+  /**
+   * Index transactions by txid.
+   * @private
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @param {CoinView} view
+   */
+
+  async indexBlock(entry, block, view) {
+    const b = this.db.batch();
+
+    for (let i = 0; i < block.txs.length; i++) {
+      const tx = block.txs[i];
+      const hash = tx.hash();
+      const meta = TXMeta.fromTX(tx, entry, i);
+      b.put(layout.t.encode(hash), meta.toRaw());
+    }
+
+    return b.write();
+  }
+
+  /**
+   * Remove transactions from index.
+   * @private
+   * @param {ChainEntry} entry
+   * @param {Block} block
+   * @param {CoinView} view
+   */
+
+  async unindexBlock(entry, block, view) {
+    const b = this.db.batch();
+
+    for (let i = 0; i < block.txs.length; i++) {
+      const tx = block.txs[i];
+      const hash = tx.hash();
+      b.del(layout.t.encode(hash));
+    }
+
+    return b.write();
+  }
+
+  /**
+   * Get a transaction with metadata.
+   * @param {Hash} hash
+   * @returns {Promise} - Returns {@link TXMeta}.
+   */
+
+  async getMeta(hash) {
+    const data = await this.db.get(layout.t.encode(hash));
+
+    if (!data)
+      return null;
+
+    return TXMeta.fromRaw(data);
+  }
+
+  /**
+   * Retrieve a transaction.
+   * @param {Hash} hash
+   * @returns {Promise} - Returns {@link TX}.
+   */
+
+  async getTX(hash) {
+    const meta = await this.getMeta(hash);
+
+    if (!meta)
+      return null;
+
+    return meta.tx;
+  }
+
+  /**
+   * @param {Hash} hash
+   * @returns {Promise} - Returns Boolean.
+   */
+
+  async hasTX(hash) {
+    return this.db.has(layout.t.encode(hash));
+  }
+
+  /**
+   * Get coin viewpoint (historical).
+   * @param {TX} tx
+   * @returns {Promise} - Returns {@link CoinView}.
+   */
+
+  async getSpentView(tx) {
+    const view = await this.client.getCoinView(tx);
+
+    for (const {prevout} of tx.inputs) {
+      if (view.hasEntry(prevout))
+        continue;
+
+      const {hash, index} = prevout;
+      const meta = await this.getMeta(hash);
+
+      if (!meta)
+        continue;
+
+      const {tx, height} = meta;
+
+      if (index < tx.outputs.length)
+        view.addIndex(tx, index, height);
+    }
+
+    return view;
+  }
+}
+
+module.exports = TXIndexer;

lib/node/fullnode.js

@@ -17,6 +17,8 @@ const Node = require('./node');
 const HTTP = require('./http');
 const RPC = require('./rpc');
 const blockstore = require('../blockstore');
+const TXIndexer = require('../indexer/txindexer');
+const AddrIndexer = require('../indexer/addrindexer');
 
 /**
  * Full Node
@@ -154,6 +156,27 @@ class FullNode extends Node {
       cors: this.config.bool('cors')
     });
 
+    // Indexers
+    this.txindex = null;
+    if (this.config.bool('index-tx'))
+      this.txindex = new TXIndexer({
+        network: this.network,
+        logger: this.logger,
+        chain: this.chain,
+        memory: this.config.bool('memory'),
+        prefix: this.config.filter('index').str('prefix') || this.config.prefix
+      });
+
+    this.addrindex = null;
+    if (this.config.bool('index-address'))
+      this.addrindex= new AddrIndexer({
+        network: this.network,
+        logger: this.logger,
+        chain: this.chain,
+        memory: this.config.bool('memory'),
+        prefix: this.config.filter('index').str('prefix') || this.config.prefix
+      });
+
     this.init();
   }
 
@@ -169,6 +192,12 @@ class FullNode extends Node {
     this.pool.on('error', err => this.error(err));
     this.miner.on('error', err => this.error(err));
 
+    if (this.txindex)
+      this.txindex.on('error', err => this.error(err));
+
+    if (this.addrindex)
+      this.addrindex.on('error', err => this.error(err));
+
     if (this.http)
       this.http.on('error', err => this.error(err));
 
@@ -235,6 +264,12 @@ class FullNode extends Node {
     await this.miner.open();
     await this.pool.open();
 
+    if (this.txindex)
+      await this.txindex.open();
+
+    if (this.addrindex)
+      await this.addrindex.open();
+
     await this.openPlugins();
 
     await this.http.open();
@@ -256,6 +291,12 @@ class FullNode extends Node {
     await this.handlePreclose();
     await this.http.close();
 
+    if (this.txindex)
+      await this.txindex.close();
+
+    if (this.addrindex)
+      await this.addrindex.close();
+
     await this.closePlugins();
 
     await this.pool.close();
@@ -417,10 +458,14 @@ class FullNode extends Node {
 
   async getCoinsByAddress(addrs) {
     const mempool = this.mempool.getCoinsByAddress(addrs);
-    const chain = await this.chain.getCoinsByAddress(addrs);
+
+    if (!this.addrindex)
+      return mempool;
+
+    const index = await this.addrindex.getCoinsByAddress(addrs);
     const out = [];
 
-    for (const coin of chain) {
+    for (const coin of index) {
       const spent = this.mempool.isSpent(coin.hash, coin.index);
 
       if (spent)
@@ -444,8 +489,23 @@ class FullNode extends Node {
 
   async getMetaByAddress(addrs) {
     const mempool = this.mempool.getMetaByAddress(addrs);
-    const chain = await this.chain.getMetaByAddress(addrs);
+
-    return chain.concat(mempool);
+    if (this.txindex && this.addrindex) {
+      if (!Array.isArray(addrs))
+        addrs = [addrs];
+
+      const hashes = await this.addrindex.getHashesByAddress(addrs);
+      const mtxs = [];
+
+      for (const hash of hashes) {
+        const mtx = await this.txindex.getMeta(hash);
+        assert(mtx);
+        mtxs.push(mtx);
+      }
+      return mtxs.concat(mempool);
+    }
+
+    return mempool;
   }
 
   /**
@@ -460,7 +520,10 @@ class FullNode extends Node {
     if (meta)
       return meta;
 
-    return this.chain.getMeta(hash);
+    if (this.txindex)
+      return this.txindex.getMeta(hash);
+
+    return null;
   }
 
   /**
@@ -472,7 +535,11 @@ class FullNode extends Node {
   async getMetaView(meta) {
     if (meta.height === -1)
       return this.mempool.getSpentView(meta.tx);
-    return this.chain.getSpentView(meta.tx);
+
+    if (this.txindex)
+      return this.txindex.getSpentView(meta.tx);
+
+    return null;
   }
 
   /**
@@ -517,7 +584,10 @@ class FullNode extends Node {
     if (this.mempool.hasEntry(hash))
       return true;
 
-    return this.chain.hasTX(hash);
+    if (this.txindex)
+      return this.txindex.hasTX(hash);
+
+    return false;
   }
 }
 

lib/node/node.js

@@ -64,6 +64,8 @@ class Node extends EventEmitter {
     this.pool = null;
     this.miner = null;
     this.http = null;
+    this.txindex = null;
+    this.addrindex = null;
 
     this._init(file);
   }

lib/node/rpc.js

@@ -963,8 +963,8 @@ class RPC extends RPCBase {
 
     if (hash) {
       block = await this.chain.getBlock(hash);
-    } else if (this.chain.options.indexTX) {
+    } else if (await this.node.hasTX(last)) {
-      const tx = await this.chain.getMeta(last);
+      const tx = await this.node.getMeta(last);
       if (tx)
         block = await this.chain.getBlock(tx.block);
     } else {

test/indexer-test.js

@@ -0,0 +1,129 @@
+/* eslint-env mocha */
+/* eslint prefer-arrow-callback: "off" */
+
+'use strict';
+
+const assert = require('./util/assert');
+const reorg = require('./util/reorg');
+const Chain = require('../lib/blockchain/chain');
+const WorkerPool = require('../lib/workers/workerpool');
+const Miner = require('../lib/mining/miner');
+const MemWallet = require('./util/memwallet');
+const TXIndexer = require('../lib/indexer/txindexer');
+const AddrIndexer = require('../lib/indexer/addrindexer');
+const Network = require('../lib/protocol/network');
+const network = Network.get('regtest');
+
+const workers = new WorkerPool({
+  enabled: true
+});
+
+const chain = new Chain({
+  memory: true,
+  network,
+  workers
+});
+
+const miner = new Miner({
+  chain,
+  version: 4,
+  workers
+});
+
+const cpu = miner.cpu;
+
+const wallet = new MemWallet({
+  network
+});
+
+const txindexer = new TXIndexer({
+  'memory': true,
+  'network': network,
+  'chain': chain
+});
+
+const addrindexer = new AddrIndexer({
+  'memory': true,
+  'network': network,
+  'chain': chain
+});
+
+describe('Indexer', function() {
+  this.timeout(45000);
+
+  it('should open indexer', async () => {
+    await chain.open();
+    await miner.open();
+    await txindexer.open();
+    await addrindexer.open();
+  });
+
+  it('should index 10 blocks', async () => {
+    miner.addresses.length = 0;
+    miner.addAddress(wallet.getReceive());
+    for (let i = 0; i < 10; i++) {
+      const block = await cpu.mineBlock();
+      assert(block);
+      assert(await chain.add(block));
+    }
+
+    assert.strictEqual(chain.height, 10);
+    assert.strictEqual(txindexer.state.startHeight, 10);
+    assert.strictEqual(addrindexer.state.startHeight, 10);
+
+    const coins =
+      await addrindexer.getCoinsByAddress(miner.getAddress());
+    assert.strictEqual(coins.length, 10);
+
+    for (const coin of coins) {
+      const meta = await txindexer.getMeta(coin.hash);
+      assert.bufferEqual(meta.tx.hash(), coin.hash);
+    }
+  });
+
+  it('should rescan and reindex 10 missed blocks', async () => {
+    await txindexer.disconnect();
+    await addrindexer.disconnect();
+
+    for (let i = 0; i < 10; i++) {
+      const block = await cpu.mineBlock();
+      assert(block);
+      assert(await chain.add(block));
+    }
+
+    assert.strictEqual(chain.height, 20);
+
+    await txindexer.connect();
+    await addrindexer.connect();
+
+    await new Promise(r => addrindexer.once('chain tip', r));
+
+    assert.strictEqual(txindexer.state.startHeight, 20);
+    assert.strictEqual(addrindexer.state.startHeight, 20);
+
+    const coins =
+      await addrindexer.getCoinsByAddress(miner.getAddress());
+    assert.strictEqual(coins.length, 20);
+
+    for (const coin of coins) {
+      const meta = await txindexer.getMeta(coin.hash);
+      assert.bufferEqual(meta.tx.hash(), coin.hash);
+    }
+  });
+
+  it('should handle indexing a reorg', async () => {
+    await reorg(chain, cpu, 10);
+
+    assert.strictEqual(txindexer.state.startHeight, 31);
+    assert.strictEqual(addrindexer.state.startHeight, 31);
+
+    const coins =
+      await addrindexer.getCoinsByAddress(miner.getAddress());
+    assert.strictEqual(coins.length, 31);
+
+    for (const coin of coins) {
+      const meta = await txindexer.getMeta(coin.hash);
+      assert.bufferEqual(meta.tx.hash(), coin.hash);
+    }
+  });
+});

test/node-test.js

@@ -28,6 +28,8 @@ const node = new FullNode({
   network: 'regtest',
   workers: true,
   plugins: [require('../lib/wallet/plugin')],
+  indexTX: true,
+  indexAddress: true,
   port: ports.p2p,
   httpPort: ports.node,
   env: {
@@ -756,6 +758,55 @@ describe('Node', function() {
     assert.strictEqual(tx1.txid(), tx2.txid());
   });
 
+  it('should get tx by hash', async () => {
+    const block = await mineBlock();
+    await chain.add(block);
+
+    const tx = block.txs[0];
+    const hash = tx.hash();
+    const hasTX = await node.hasTX(hash);
+
+    assert.strictEqual(hasTX, true);
+
+    const tx2 = await node.getTX(hash);
+    assert.strictEqual(tx.txid(), tx2.txid());
+
+    const meta = await node.getMeta(hash);
+    assert.strictEqual(meta.tx.txid(), tx2.txid());
+  });
+
+  it('should get coin/tx by addr', async () => {
+    const addr = await wallet.receiveAddress();
+    const mtx = await wallet.createTX({
+      rate: 100000,
+      outputs: [{
+        value: 100000,
+        address: addr
+      }]
+    });
+
+    await wallet.sign(mtx);
+
+    const tx = mtx.toTX();
+    const job = await miner.createJob();
+
+    job.addTX(tx, mtx.view);
+    job.refresh();
+
+    const block = await job.mineAsync();
+    await chain.add(block);
+
+    await new Promise(r => setTimeout(r, 300));
+
+    const txs = await node.getTXByAddress(addr.hash);
+    const tx2 = txs[0];
+    assert.strictEqual(tx.txid(), tx2.txid());
+
+    const coins = await node.getCoinsByAddress(addr.hash);
+    const coin = coins[0];
+    assert.strictEqual(tx.txid(), coin.txid());
+  });
+
   it('should cleanup', async () => {
     consensus.COINBASE_MATURITY = 100;
     await node.close();

test/util/reorg.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const assert = require('./assert');
+const Chain = require('../../lib/blockchain/chain');
+const CPUMiner = require('../../lib/mining/cpuminer');
+
+/**
+ * Reorgs the chain to given height using miners.
+ * @param {Chain} chain chain
+ * @param {CPUMiner} cpu cpuminer
+ * @param {Number} height height
+ * @returns {Promise} null
+ */
+async function reorg(chain, cpu, height) {
+  assert(chain instanceof Chain);
+  assert(cpu instanceof CPUMiner);
+  assert(typeof height === 'number');
+
+  let tip1, tip2 = null;
+  for (let i = 0; i < height; i++) {
+    const job1 = await cpu.createJob(tip1);
+    const job2 = await cpu.createJob(tip2);
+
+    const blk1 = await job1.mineAsync();
+    const blk2 = await job2.mineAsync();
+
+    const hash1 = blk1.hash();
+    const hash2 = blk2.hash();
+
+    assert(await chain.add(blk1));
+    assert(await chain.add(blk2));
+
+    assert.bufferEqual(chain.tip.hash, hash1);
+
+    tip1 = await chain.getEntry(hash1);
+    tip2 = await chain.getEntry(hash2);
+
+    assert(tip1);
+    assert(tip2);
+
+    assert(!await chain.isMainChain(tip2));
+  }
+
+  const entry = await chain.getEntry(tip2.hash);
+  assert(entry);
+  assert.strictEqual(chain.height, entry.height);
+
+  const block = await cpu.mineBlock(entry);
+  assert(block);
+
+  let forked = false;
+  chain.once('reorganize', () => {
+    forked = true;
+  });
+
+  assert(await chain.add(block));
+
+  assert(forked);
+  assert.bufferEqual(chain.tip.hash, block.hash());
+  assert(chain.tip.chainwork.gt(tip1.chainwork));
+}
+
+module.exports = reorg;