indexer: use chain and blocks for indexer, remove chain client

lib/bcoin-browser.js

@@ -58,7 +58,6 @@ bcoin.Mnemonic = require('./hd/mnemonic');
 // Index
 bcoin.indexer = require('./indexer');
 bcoin.Indexer = require('./indexer/indexer');
-bcoin.ChainClient = require('./indexer/chainclient');
 bcoin.TXIndexer = require('./indexer/txindexer');
 bcoin.AddrIndexer = require('./indexer/addrindexer');
 

lib/bcoin.js

@@ -79,7 +79,6 @@ bcoin.define('Mnemonic', './hd/mnemonic');
 // Index
 bcoin.define('indexer', './indexer');
 bcoin.define('Indexer', './indexer/indexer');
-bcoin.define('ChainClient', './indexer/chainclient');
 bcoin.define('TXIndexer', './indexer/txindexer');
 bcoin.define('AddrIndexer', './indexer/addrindexer');
 

lib/indexer/addrindexer.js

@@ -159,7 +159,7 @@ class AddrIndexer extends Indexer {
       });
 
       for (const [hash, index] of keys) {
-        const coin = await this.client.getCoin(hash, index);
+        const coin = await this.chain.getCoin(hash, index);
         assert(coin);
         coins.push(coin);
       }

lib/indexer/chainclient.js

@@ -1,200 +0,0 @@
-/*!
- * chainclient.js - chain client for bcoin
- * Copyright (c) 2018, the bcoin developers (MIT License).
- * https://github.com/bcoin-org/bcoin
- */
-
-'use strict';
-
-const assert = require('assert');
-const AsyncEmitter = require('bevent');
-const Chain = require('../blockchain/chain');
-
-/**
- * Chain Client
- * @extends AsyncEmitter
- * @alias module:indexer.ChainClient
- */
-
-class ChainClient extends AsyncEmitter {
-  /**
-   * Create a chain client.
-   * @constructor
-   * @param {Chain} chain
-   */
-
-  constructor(chain) {
-    super();
-
-    assert(chain instanceof Chain);
-
-    this.chain = chain;
-    this.network = chain.network;
-    this.opened = false;
-
-    this.init();
-  }
-
-  /**
-   * Initialize the client.
-   */
-
-  init() {
-    this.chain.on('connect', async (entry, block, view) => {
-      if (!this.opened)
-        return;
-
-      await this.emitAsync('block connect', entry, block, view);
-    });
-
-    this.chain.on('disconnect', async (entry, block, view) => {
-      if (!this.opened)
-        return;
-
-      await this.emitAsync('block disconnect', entry, block, view);
-    });
-
-    this.chain.on('reset', async (tip) => {
-      if (!this.opened)
-        return;
-
-      await this.emitAsync('chain reset', tip);
-    });
-  }
-
-  /**
-   * Open the client.
-   * @returns {Promise}
-   */
-
-  async open(options) {
-    assert(!this.opened, 'ChainClient is already open.');
-    this.opened = true;
-    setImmediate(() => this.emit('connect'));
-  }
-
-  /**
-   * Close the client.
-   * @returns {Promise}
-   */
-
-  async close() {
-    assert(this.opened, 'ChainClient is not open.');
-    this.opened = false;
-    setImmediate(() => this.emit('disconnect'));
-  }
-
-  /**
-   * Get chain tip.
-   * @returns {Promise}
-   */
-
-  async getTip() {
-    return this.chain.tip;
-  }
-
-  /**
-   * Get chain entry.
-   * @param {Hash} hash
-   * @returns {Promise} - Returns {@link ChainEntry}.
-   */
-
-  async getEntry(hash) {
-    const entry = await this.chain.getEntry(hash);
-
-    if (!entry)
-      return null;
-
-    if (!await this.chain.isMainChain(entry))
-      return null;
-
-    return entry;
-  }
-
-  /**
-   * Get a coin (unspents only).
-   * @param {Hash} hash
-   * @param {Number} index
-   * @returns {Promise} - Returns {@link Coin}.
-   */
-
-  async getCoin(hash, index) {
-    return this.chain.getCoin(hash, index);
-  }
-
-  /**
-   * Get hash range.
-   * @param {Number} start
-   * @param {Number} end
-   * @returns {Promise}
-   */
-
-  async getHashes(start = -1, end = -1) {
-    return this.chain.getHashes(start, end);
-  }
-
-  /**
-   * Get block
-   * @param {Hash} hash
-   * @returns {Promise} - Returns {@link Block}
-   */
-
-  async getBlock(hash) {
-    const block = await this.chain.getBlock(hash);
-
-    if (!block)
-      return null;
-
-    return block;
-  }
-
-  /**
-   * Get a historical block coin viewpoint.
-   * @param {Block} hash
-   * @returns {Promise} - Returns {@link CoinView}.
-   */
-
-  async getBlockView(block) {
-    return this.chain.getBlockView(block);
-  }
-
-  /**
-   * Get coin viewpoint.
-   * @param {TX} tx
-   * @returns {Promise} - Returns {@link CoinView}.
-   */
-
-  async getCoinView(tx) {
-    return this.chain.getCoinView(tx);
-  }
-
-  /**
-   * Rescan for any missed blocks.
-   * @param {Number} start - Start block.
-   * @returns {Promise}
-   */
-
-  async rescan(start) {
-    for (let i = start; ; i++) {
-      const entry = await this.getEntry(i);
-      if (!entry) {
-        await this.emitAsync('chain tip');
-        break;
-      };
-
-      const block = await this.getBlock(entry.hash);
-      assert(block);
-
-      const view = await this.getBlockView(block);
-      assert(view);
-
-      await this.emitAsync('block rescan', entry, block, view);
-    }
-  };
-}
-
-/*
- * Expose
- */
-
-module.exports = ChainClient;

lib/indexer/index.js

@@ -13,4 +13,3 @@
 exports.Indexer = require('./indexer');
 exports.TXIndexer = require('./txindexer');
 exports.AddrIndexer = require('./addrindexer');
-exports.ChainClient = require('./chainclient');

lib/indexer/indexer.js

@@ -15,8 +15,6 @@ const Logger = require('blgr');
 const Network = require('../protocol/network');
 const layout = require('./layout');
 const records = require('./records');
-const ChainClient = require('./chainclient');
-const NullClient = require('./nullclient');
 
 const {
   ChainState,
@@ -51,7 +49,9 @@ class Indexer extends EventEmitter {
 
     this.network = this.options.network;
     this.logger = this.options.logger.context(`${module}indexer`);
-    this.client = this.options.client || new NullClient(this);
+    this.blocks = this.options.blocks;
+    this.chain = this.options.chain;
+
     this.db = null;
     this.rescanning = false;
 
@@ -59,17 +59,6 @@ class Indexer extends EventEmitter {
     this.height = 0;
 
     this.lock = new Lock();
-
-    this.init();
-  }
-
-  /**
-   * Initialize indexdb.
-   * @private
-   */
-
-  init() {
-    this._bind();
   }
 
   /**
@@ -77,22 +66,11 @@ class Indexer extends EventEmitter {
    * @private
    */
 
-  _bind() {
+  bind() {
-    this.client.on('error', (err) => {
+    this.chain.on('connect', async (entry, block, view) => {
-      this.emit('error', err);
-    });
-
-    this.client.on('connect', async () => {
-      try {
-        await this.syncNode();
-      } catch (e) {
-        this.emit('error', e);
-      }
-    });
-
-    this.client.on('block connect', async (entry, block, view) => {
       if (this.rescanning)
         return;
+
       try {
         await this.addBlock(entry, block, view);
       } catch (e) {
@@ -100,9 +78,10 @@ class Indexer extends EventEmitter {
       }
     });
 
-    this.client.on('block disconnect', async (entry, block, view) => {
+    this.chain.on('disconnect', async (entry, block, view) => {
       if (this.rescanning)
         return;
+
       try {
         await this.removeBlock(entry, block, view);
       } catch (e) {
@@ -110,27 +89,13 @@ class Indexer extends EventEmitter {
       }
     });
 
-    this.client.on('block rescan', async (entry, block, view) => {
+    this.chain.on('reset', async (tip) => {
-      try {
-        await this.rescanBlock(entry, block, view);
-      } catch (e) {
-        this.emit('error', e);
-      }
-    });
-
-    this.client.on('chain reset', async (tip) => {
       try {
         await this.resetChain(tip);
       } catch (e) {
         this.emit('error', e);
       }
     });
-
-    this.client.on('chain tip', async () => {
-      this.logger.debug('Indexer: finished rescan');
-      const tip = await this.getTip();
-      this.emit('chain tip', tip);
-    });
   }
 
   /**
@@ -160,7 +125,9 @@ class Indexer extends EventEmitter {
 
     await this.verifyNetwork();
 
-    await this.connect();
+    this.bind();
+
+    await this.sync();
   }
 
   /**
@@ -191,34 +158,15 @@ class Indexer extends EventEmitter {
    */
 
   async close() {
-    await this.disconnect();
     return this.db.close();
   }
 
-  /**
-   * Connect to the chain server (client required).
-   * @returns {Promise}
-   */
-
-  async connect() {
-    return this.client.open();
-  }
-
-  /**
-   * Disconnect from chain server (client required).
-   * @returns {Promise}
-   */
-
-  async disconnect() {
-    return this.client.close();
-  }
-
   /**
    * Sync state with server on every connect.
    * @returns {Promise}
    */
 
-  async syncNode() {
+  async sync() {
     const unlock = await this.lock.lock();
     try {
       this.logger.info('Resyncing from server...');
@@ -251,7 +199,7 @@ class Indexer extends EventEmitter {
     this.logger.info('Initializing database state from server.');
 
     const b = this.db.batch();
-    const hashes = await this.client.getHashes();
+    const hashes = await this.chain.getHashes();
 
     let tip = null;
 
@@ -278,6 +226,24 @@ class Indexer extends EventEmitter {
     return undefined;
   }
 
+  /**
+   * Get a chain entry for the main chain only.
+   * @private
+   * @returns {Promise}
+   */
+
+  async getEntry(hash) {
+    const entry = await this.chain.getEntry(hash);
+
+    if (!entry)
+      return null;
+
+    if (!await this.chain.isMainChain(entry))
+      return null;
+
+    return entry;
+  }
+
   /**
    * Connect and sync with the chain server.
    * @private
@@ -289,30 +255,29 @@ class Indexer extends EventEmitter {
 
     this.logger.info('Syncing state from height %d.', height);
 
-    // re-org when we're offline might
+    // A re-org when we're offline might
-    // leave chain in different state.
+    // leave chain in a different state.
-    // scan chain backwards until we
+    // Scan chain backwards until we
-    // find a known 'good' height
+    // find a known 'good' height.
     for (;;) {
       const tip = await this.getBlock(height);
       assert(tip);
 
-      if (await this.client.getEntry(tip.hash))
+      if (await this.getEntry(tip.hash))
         break;
 
       assert(height !== 0);
       height -= 1;
     }
 
-    // start scan from last indexed OR
+    // Start scan from last indexed OR
     // last known 'good' height whichever
     // is lower, because `scan` scans from
     // low to high blocks
     if (this.state.startHeight < height)
       height = this.state.startHeight;
 
-    this.logger.spam('Starting block rescan from: %d.', height);
+    return this._rescan(height);
-    return this.scan(height);
   }
 
   /**
@@ -332,7 +297,7 @@ class Indexer extends EventEmitter {
     }
 
     if (entry.height % 1000 === 0)
-      this.logger.debug('rescanned block: %d.', entry.height);
+      this.logger.debug('Rescanned block: %d.', entry.height);
 
     if (entry.height > this.state.height + 1) {
       this.logger.warning('Rescan block too high: %d.', entry.height);
@@ -347,33 +312,6 @@ class Indexer extends EventEmitter {
     }
   }
 
-  /**
-   * Rescan blockchain from a given height.
-   * @private
-   * @param {Number?} height
-   * @returns {Promise}
-   */
-
-  async scan(height) {
-    assert((height >>> 0) === height, 'Indexer: Must pass in a height.');
-
-    await this.rollback(height);
-
-    const tip = this.state.height;
-
-    this.logger.info(
-      'Indexer is scanning %d blocks.',
-      tip - height + 1);
-
-    try {
-      this.rescanning = true;
-      this.logger.debug('rescanning from %d to %d', height, tip);
-      await this.client.rescan(height);
-    } finally {
-      this.rescanning = false;
-    }
-  }
-
   /**
    * Force a rescan.
    * @param {Number} height
@@ -390,14 +328,38 @@ class Indexer extends EventEmitter {
   }
 
   /**
-   * Force a rescan (without a lock).
+   * Rescan blockchain from a given height.
    * @private
    * @param {Number} height
    * @returns {Promise}
    */
 
   async _rescan(height) {
-    return this.scan(height);
+    assert((height >>> 0) === height, 'Must pass in a height.');
+
+    await this.rollback(height);
+
+    const tip = this.state.height;
+
+    this.logger.debug('Rescanning from %d to %d', height, tip);
+
+    this.rescanning = true;
+
+    for (let i = height; ; i++) {
+      const entry = await this.chain.getEntry(i);
+      if (!entry)
+        break;
+
+      const block = await this.chain.getBlock(entry.hash);
+      assert(block);
+
+      const view = await this.chain.getBlockView(block);
+      assert(view);
+
+      await this.rescanBlock(entry, block, view);
+    }
+
+    this.rescanning = false;
   }
 
   /**
@@ -551,7 +513,7 @@ class Indexer extends EventEmitter {
       // processed (in the case of a crash).
       this.logger.warning('Already saw Indexer block (%d).', tip.height);
     } else if (tip.height !== this.state.startHeight + 1) {
-      await this.scan(this.state.height);
+      await this._rescan(this.state.height);
       return;
     }
 
@@ -700,7 +662,7 @@ class IndexOptions {
     this.module = module;
     this.network = Network.primary;
     this.logger = Logger.global;
-    this.client = null;
+    this.blocks = null;
     this.chain = null;
     this.indexers = null;
 
@@ -723,6 +685,14 @@ class IndexOptions {
    */
 
   fromOptions(options) {
+    assert(options.blocks && typeof options.blocks === 'object',
+           'Indexer requires a blockstore.');
+    assert(options.chain && typeof options.chain === 'object',
+           'Indexer requires chain.');
+
+    this.blocks = options.blocks;
+    this.chain = options.chain;
+
     if (options.network != null)
       this.network = Network.get(options.network);
 
@@ -731,20 +701,6 @@ class IndexOptions {
       this.logger = options.logger;
     }
 
-    if (options.client != null) {
-      assert(typeof options.client === 'object');
-      this.client = options.client;
-    }
-
-    if (options.chain != null) {
-      assert(typeof options.chain === 'object');
-      this.client = new ChainClient(options.chain);
-    }
-
-    if (!this.client) {
-      throw new Error('Client is required');
-    }
-
     if (options.prefix != null) {
       assert(typeof options.prefix === 'string');
       this.prefix = options.prefix;

lib/indexer/nullclient.js

@@ -1,142 +0,0 @@
-/*!
- * nullclient.js - chain client for bcoin
- * Copyright (c) 2018, the bcoin developers (MIT License).
- * https://github.com/bcoin-org/bcoin
- */
-
-'use strict';
-
-const assert = require('assert');
-const EventEmitter = require('events');
-
-/**
- * Null Client
- * Sort of a fake local client for separation of concerns.
- * @alias module:indexer.NullClient
- */
-
-class NullClient extends EventEmitter {
-  /**
-   * Create a client.
-   * @constructor
-   * @param {Chain} chain
-   */
-
-  constructor(chain) {
-    super();
-
-    this.chain = chain;
-    this.network = chain.network;
-    this.opened = false;
-  }
-
-  /**
-   * Open the client.
-   * @returns {Promise}
-   */
-
-  async open(options) {
-    assert(!this.opened, 'NullClient is already open.');
-    this.opened = true;
-    setImmediate(() => this.emit('connect'));
-  }
-
-  /**
-   * Close the client.
-   * @returns {Promise}
-   */
-
-  async close() {
-    assert(this.opened, 'NullClient is not open.');
-    this.opened = false;
-    setImmediate(() => this.emit('disconnect'));
-  }
-
-  /**
-   * Get chain tip.
-   * @returns {Promise}
-   */
-
-  async getTip() {
-    const {hash, height, time} = this.network.genesis;
-    return { hash, height, time };
-  }
-
-  /**
-   * Get chain entry.
-   * @param {Hash} hash
-   * @returns {Promise} - Returns {@link ChainEntry}.
-   */
-
-  async getEntry(hash) {
-    return { hash, height: 0, time: 0 };
-  }
-
-  /**
-   * Get a coin (unspents only).
-   * @param {Hash} hash
-   * @param {Number} index
-   * @returns {Promise} - Returns {@link Coin}.
-   */
-
-  async getCoin(hash, index) {
-    return null;
-  }
-
-  /**
-   * Get hash range.
-   * @param {Number} start
-   * @param {Number} end
-   * @returns {Promise}
-   */
-
-  async getHashes(start = -1, end = -1) {
-    return [this.network.genesis.hash];
-  }
-
-  /**
-   * Get block
-   * @param {Hash} hash
-   * @returns {Promise}
-   */
-
-  async getBlock(hash) {
-    return null;
-  }
-
-  /**
-   * Get a historical block coin viewpoint.
-   * @param {Block} hash
-   * @returns {Promise} - Returns {@link CoinView}.
-   */
-
-  async getBlockView(block) {
-    return null;
-  }
-
-  /**
-   * Get coin viewpoint.
-   * @param {TX} tx
-   * @returns {Promise} - Returns {@link CoinView}.
-   */
-
-  async getCoinView(tx) {
-    return null;
-  }
-
-  /**
-   * Rescan for any missed blocks.
-   * @param {Number} start - Start block.
-   * @returns {Promise}
-   */
-
-  async rescan(start) {
-    ;
-  }
-}
-
-/*
- * Expose
- */
-
-module.exports = NullClient;

lib/indexer/txindexer.js

@@ -184,8 +184,10 @@ class TXIndexer extends Indexer {
       return null;
 
     const record = TxRecord.fromRaw(raw);
+    const {block, offset, length} = record;
+
+    const data = await this.blocks.read(block, offset, length);
 
-    const data = await this.read(record.block, record.offset, record.length);
     const tx = TX.fromRaw(data);
 
     const meta = TXMeta.fromTX(tx);
@@ -228,7 +230,7 @@ class TXIndexer extends Indexer {
    */
 
   async getSpentView(tx) {
-    const view = await this.client.getCoinView(tx);
+    const view = await this.chain.getCoinView(tx);
 
     for (const {prevout} of tx.inputs) {
       if (view.hasEntry(prevout))

lib/node/fullnode.js

@@ -157,25 +157,27 @@ class FullNode extends Node {
     });
 
     // Indexers
-    this.txindex = null;
+    if (this.config.bool('index-tx')) {
-    if (this.config.bool('index-tx'))
       this.txindex = new TXIndexer({
         network: this.network,
         logger: this.logger,
+        blocks: this.blocks,
         chain: this.chain,
         memory: this.config.bool('memory'),
         prefix: this.config.filter('index').str('prefix') || this.config.prefix
       });
+    }
 
-    this.addrindex = null;
+    if (this.config.bool('index-address')) {
-    if (this.config.bool('index-address'))
       this.addrindex= new AddrIndexer({
         network: this.network,
         logger: this.logger,
+        blocks: this.blocks,
         chain: this.chain,
         memory: this.config.bool('memory'),
         prefix: this.config.filter('index').str('prefix') || this.config.prefix
       });
+    }
 
     this.init();
   }

test/indexer-test.js

@@ -44,15 +44,17 @@ const wallet = new MemWallet({
 });
 
 const txindexer = new TXIndexer({
-  'memory': true,
+  memory: true,
-  'network': network,
+  network,
-  'chain': chain
+  chain,
+  blocks
 });
 
 const addrindexer = new AddrIndexer({
-  'memory': true,
+  memory: true,
-  'network': network,
+  network,
-  'chain': chain
+  chain,
+  blocks
 });
 
 describe('Indexer', function() {
@@ -90,9 +92,6 @@ describe('Indexer', function() {
   });
 
   it('should rescan and reindex 10 missed blocks', async () => {
-    await txindexer.disconnect();
-    await addrindexer.disconnect();
-
     for (let i = 0; i < 10; i++) {
       const block = await cpu.mineBlock();
       assert(block);
@@ -100,17 +99,10 @@ describe('Indexer', function() {
     }
 
     assert.strictEqual(chain.height, 20);
-
-    await txindexer.connect();
-    await addrindexer.connect();
-
-    await new Promise(r => addrindexer.once('chain tip', r));
-
     assert.strictEqual(txindexer.state.startHeight, 20);
     assert.strictEqual(addrindexer.state.startHeight, 20);
 
-    const coins =
+    const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
-      await addrindexer.getCoinsByAddress(miner.getAddress());
     assert.strictEqual(coins.length, 20);
 
     for (const coin of coins) {