indexer: fix memory and cpu exhaustion for addrindex

lib/indexer/addrindexer.js

@@ -8,6 +8,7 @@
 
 const assert = require('assert');
 const bdb = require('bdb');
+const bio = require('bufio');
 const {BufferSet} = require('buffer-map');
 const layout = require('./layout');
 const Address = require('../primitives/address');
@@ -15,15 +16,93 @@ const Indexer = require('./indexer');
 
 /*
  * AddrIndexer Database Layout:
- *  T[addr-hash][hash] -> dummy (tx by address)
+ *  T[addr-hash][height][tx-index][hash] -> dummy (tx by address)
- *  C[addr-hash][hash][index] -> dummy (coin by address)
+ *  C[addr-hash][height][tx-index][hash][coin-index] -> dummy (coin by address)
+ *  x[addr-hash][hash] -> height and tx-index for tx
+ *  y[addr-hash][hash][index] -> height, tx-index and coin-index for coin
 */
 
 Object.assign(layout, {
-  T: bdb.key('T', ['hash', 'hash256']),
+  T: bdb.key('T', ['hash', 'uint32', 'uint32', 'hash256']),
-  C: bdb.key('C', ['hash', 'hash256', 'uint32'])
+  C: bdb.key('C', ['hash', 'uint32', 'uint32', 'hash256', 'uint32']),
+  x: bdb.key('x', ['hash', 'hash256']),
+  y: bdb.key('y', ['hash', 'hash256', 'uint32'])
 });
 
+/**
+ * Count
+ */
+
+class Count {
+  /**
+   * Create count record.
+   * @constructor
+   * @param {Number} height
+   * @param {Number} index
+   */
+
+  constructor(height, index, coin) {
+    this.height = height >= 0 ? height : 0;
+    this.index = index >= 0 ? index : 0;
+    this.coin = coin >= 0 ? coin : -1;
+
+    assert((this.height >>> 0) === this.height);
+    assert((this.index >>> 0) === this.index);
+
+    if (coin)
+      assert((this.coin >>> 0) === this.coin);
+  }
+
+  /**
+   * Serialize.
+   * @returns {Buffer}
+   */
+
+  toRaw() {
+    let len = 8;
+    if (this.coin >= 0)
+      len += 4;
+
+    const bw = bio.write(len);
+
+    bw.writeU32(this.height);
+    bw.writeU32(this.index);
+
+    if (this.coin >= 0)
+      bw.writeU32(this.coin);
+
+    return bw.render();
+  }
+
+  /**
+   * Deserialize.
+   * @private
+   * @param {Buffer} data
+   */
+
+  fromRaw(data) {
+    const br = bio.read(data);
+
+    this.height = br.readU32();
+    this.index = br.readU32();
+
+    if (br.left() >= 4)
+      this.coin = br.readU32();
+
+    return this;
+  }
+
+  /**
+   * Instantiate a count from a buffer.
+   * @param {Buffer} data
+   * @returns {Count}
+   */
+
+  static fromRaw(data) {
+    return new this().fromRaw(data);
+  }
+}
+
 /**
  * AddrIndexer
  * @alias module:indexer.AddrIndexer
@@ -41,6 +120,8 @@ class AddrIndexer extends Indexer {
     super('addr', options);
 
     this.db = bdb.create(this.options);
+    this.maxTxs = options.maxTxs || 100;
+    this.maxCoins = options.maxCoins || 500;
   }
 
   /**
@@ -53,12 +134,18 @@ class AddrIndexer extends Indexer {
 
   async indexBlock(entry, block, view) {
     const b = this.db.batch();
+    const height = entry.height;
 
     for (let i = 0; i < block.txs.length; i++) {
       const tx = block.txs[i];
       const hash = tx.hash();
-      for (const addr of tx.getHashes(view))
+
-        b.put(layout.T.encode(addr, hash), null);
+      for (const addr of tx.getHashes(view)) {
+        const count = new Count(height, i);
+
+        b.put(layout.T.encode(addr, height, i, hash), null);
+        b.put(layout.x.encode(addr, hash), count.toRaw());
+      }
 
       if (!tx.isCoinbase()) {
         for (const {prevout} of tx.inputs) {
@@ -71,18 +158,22 @@ class AddrIndexer extends Indexer {
           if (!addr)
             continue;
 
-          b.del(layout.C.encode(addr, hash, index));
+          b.del(layout.C.encode(addr, height, i, hash, index));
+          b.del(layout.y.encode(addr, hash, index));
         }
       }
 
-      for (let i = 0; i < tx.outputs.length; i++) {
+      for (let j = 0; j < tx.outputs.length; j++) {
-        const output = tx.outputs[i];
+        const output = tx.outputs[j];
         const addr = output.getHash();
 
         if (!addr)
           continue;
 
-        b.put(layout.C.encode(addr, hash, i), null);
+        const count = new Count(height, i, j);
+
+        b.put(layout.C.encode(addr, height, i, hash, j), null);
+        b.put(layout.y.encode(addr, hash, j), count.toRaw());
       }
     }
 
@@ -99,11 +190,16 @@ class AddrIndexer extends Indexer {
 
   async unindexBlock(entry, block, view) {
     const b = this.db.batch();
+    const height = entry.height;
+
     for (let i = 0; i < block.txs.length; i++) {
       const tx = block.txs[i];
       const hash = tx.hash();
-      for (const addr of tx.getHashes(view))
+
-        b.del(layout.T.encode(addr, hash));
+      for (const addr of tx.getHashes(view)) {
+        b.del(layout.T.encode(addr, height, i, hash));
+        b.del(layout.x.encode(addr, hash));
+      }
 
       if (!tx.isCoinbase()) {
         for (const {prevout} of tx.inputs) {
@@ -116,18 +212,22 @@ class AddrIndexer extends Indexer {
           if (!addr)
             continue;
 
-          b.put(layout.C.encode(addr, hash, index), null);
+          const count = new Count(height, i);
+
+          b.put(layout.C.encode(addr, height, i, hash, index), null);
+          b.put(layout.y.encode(addr, hash, index), count.toRaw());
         }
       }
 
-      for (let i = 0; i < tx.outputs.length; i++) {
+      for (let j = 0; j < tx.outputs.length; j++) {
-        const output = tx.outputs[i];
+        const output = tx.outputs[j];
         const addr = output.getHash();
 
         if (!addr)
           continue;
 
-        b.del(layout.C.encode(addr, hash, i));
+        b.del(layout.C.encode(addr, height, i, hash, j));
+        b.del(layout.y.encode(addr, hash, j));
       }
     }
 
@@ -136,33 +236,103 @@ class AddrIndexer extends Indexer {
 
   /**
    * Get all coins pertinent to an address.
-   * @param {Address[]} addrs
+   * @param {Address} addr
+   * @param {Object} options
+   * @param {Boolean} options.reverse
+   * @param {Boolean} options.limit
    * @returns {Promise} - Returns {@link Coin}[].
    */
 
-  async getCoinsByAddress(addrs) {
+  async getCoinsByAddress(addr, options = {}) {
-    if (!Array.isArray(addrs))
-      addrs = [addrs];
-
     const coins = [];
 
-    for (const addr of addrs) {
+    const {reverse} = options;
-      const hash = Address.getHash(addr);
+    let {limit} = options;
 
-      const keys = await this.db.keys({
+    if (!limit)
-        gte: layout.C.min(hash),
+      limit = this.maxCoins;
-        lte: layout.C.max(hash),
-        parse: (key) => {
-          const [, txid, index] = layout.C.decode(key);
-          return [txid, index];
-        }
-      });
 
-      for (const [hash, index] of keys) {
+    if (limit > this.maxCoins)
-        const coin = await this.chain.getCoin(hash, index);
+      throw new Error('Limit above max of ${this.maxCoins}.');
-        assert(coin);
+
-        coins.push(coin);
+    const hash = Address.getHash(addr);
+
+    const keys = await this.db.keys({
+      gte: layout.C.min(hash),
+      lte: layout.C.max(hash),
+      limit,
+      reverse,
+      parse: (key) => {
+        const [,,, txid, index] = layout.C.decode(key);
+        return [txid, index];
       }
+    });
+
+    for (const [hash, index] of keys) {
+      const coin = await this.chain.getCoin(hash, index);
+      assert(coin);
+      coins.push(coin);
+    }
+
+    return coins;
+  }
+
+  /**
+   * Get all coins pertinent to an address after a
+   * specific txid and output/coin index.
+   * @param {Address} addr
+   * @param {Object} options
+   * @param {Buffer} options.txid
+   * @param {Number} options.index
+   * @param {Boolean} options.limit
+   * @param {Boolean} options.reverse
+   * @returns {Promise} - Returns {@link Coin}[].
+   */
+
+  async getCoinsByAddressAfter(addr, options = {}) {
+    const coins = [];
+
+    const {txid, index, reverse} = options;
+    let {limit} = options;
+
+    if (!limit)
+      limit = this.maxCoins;
+
+    if (limit > this.maxCoins)
+      throw new Error('Limit above max of ${this.maxCoins}.');
+
+    const hash = Address.getHash(addr);
+
+    const raw = await this.db.get(layout.y.encode(hash, txid, index));
+
+    if (!raw)
+      return coins;
+
+    const count = Count.fromRaw(raw);
+
+    const opts = {
+      limit,
+      reverse,
+      parse: (key) => {
+        const [,,, txid, index] = layout.C.decode(key);
+        return [txid, index];
+      }
+    };
+
+    if (!reverse) {
+      opts.gt = layout.C.min(hash, count.height, count.index, txid, count.coin);
+      opts.lte = layout.C.max(hash);
+    } else {
+      opts.gte = layout.C.min(hash);
+      opts.lt = layout.C.max(hash, count.height, count.index, txid, count.coin);
+    }
+
+    const keys = await this.db.keys(opts);
+
+    for (const [hash, index] of keys) {
+      const coin = await this.chain.getCoin(hash, index);
+      assert(coin);
+      coins.push(coin);
     }
 
     return coins;
@@ -170,26 +340,93 @@ class AddrIndexer extends Indexer {
 
   /**
    * Get all transaction hashes to an address.
-   * @param {Address[]} addrs
+   * @param {Address} addr
+   * @param {Object} options
+   * @param {Boolean} options.limit
+   * @param {Boolean} options.reverse
    * @returns {Promise} - Returns {@link Hash}[].
    */
 
-  async getHashesByAddress(addrs) {
+  async getHashesByAddress(addr, options = {}) {
     const set = new BufferSet();
 
-    for (const addr of addrs) {
+    const {reverse} = options;
-      const hash = Address.getHash(addr);
+    let {limit} = options;
 
-      await this.db.keys({
+    if (!limit)
-        gte: layout.T.min(hash),
+      limit = this.maxTxs;
-        lte: layout.T.max(hash),
+
-        parse: (key) => {
+    if (limit > this.maxTxs)
-          const [, txid] = layout.T.decode(key);
+      throw new Error('Limit above max of ${this.maxTxs}.');
-          set.add(txid);
+
-        }
+    const hash = Address.getHash(addr);
-      });
+
+    await this.db.keys({
+      gte: layout.T.min(hash),
+      lte: layout.T.max(hash),
+      limit,
+      reverse,
+      parse: (key) => {
+        const [,,, txid] = layout.T.decode(key);
+        set.add(txid);
+      }
+    });
+
+    return set.toArray();
+  }
+
+  /**
+   * Get all transaction hashes to an address after
+   * a specific txid.
+   * @param {Address} addr
+   * @param {Object} options
+   * @param {Buffer} options.txid
+   * @param {Boolean} options.limit
+   * @param {Boolean} options.reverse
+   * @returns {Promise} - Returns {@link Hash}[].
+   */
+
+  async getHashesByAddressAfter(addr, options = {}) {
+    const set = new BufferSet();
+
+    const hash = Address.getHash(addr);
+
+    const {txid, reverse} = options;
+    let {limit} = options;
+
+    if (!limit)
+      limit = this.maxTxs;
+
+    if (limit > this.maxTxs)
+      throw new Error('Limit above max of ${this.maxTxs}.');
+
+    const raw = await this.db.get(layout.x.encode(hash, txid));
+
+    if (!raw)
+      return [];
+
+    const count = Count.fromRaw(raw);
+    const {height, index} = count;
+
+    const opts = {
+      limit,
+      reverse,
+      parse: (key) => {
+        const [,,, txid] = layout.T.decode(key);
+        set.add(txid);
+      }
+    };
+
+    if (!reverse) {
+      opts.gt = layout.T.min(hash, height, index, txid);
+      opts.lte = layout.T.max(hash);
+    } else {
+      opts.gte = layout.T.min(hash);
+      opts.lt = layout.T.max(hash, height, index, txid);
     }
 
+    await this.db.keys(opts);
+
     return set.toArray();
   }
 }

lib/node/fullnode.js

@@ -454,17 +454,17 @@ class FullNode extends Node {
   /**
    * Get coins that pertain to an address from the mempool or chain database.
    * Takes into account spent coins in the mempool.
-   * @param {Address} addrs
+   * @param {Address} addr
    * @returns {Promise} - Returns {@link Coin}[].
    */
 
-  async getCoinsByAddress(addrs) {
+  async getCoinsByAddress(addr) {
-    const mempool = this.mempool.getCoinsByAddress(addrs);
+    const mempool = this.mempool.getCoinsByAddress(addr);
 
     if (!this.addrindex)
       return mempool;
 
-    const index = await this.addrindex.getCoinsByAddress(addrs);
+    const index = await this.addrindex.getCoinsByAddress(addr);
     const out = [];
 
     for (const coin of index) {
@@ -485,18 +485,15 @@ class FullNode extends Node {
   /**
    * Retrieve transactions pertaining to an
    * address from the mempool or chain database.
-   * @param {Address} addrs
+   * @param {Address} addr
    * @returns {Promise} - Returns {@link TXMeta}[].
    */
 
-  async getMetaByAddress(addrs) {
+  async getMetaByAddress(addr) {
-    const mempool = this.mempool.getMetaByAddress(addrs);
+    const mempool = this.mempool.getMetaByAddress(addr);
 
     if (this.txindex && this.addrindex) {
-      if (!Array.isArray(addrs))
+      const hashes = await this.addrindex.getHashesByAddress(addr);
-        addrs = [addrs];
-
-      const hashes = await this.addrindex.getHashesByAddress(addrs);
       const mtxs = [];
 
       for (const hash of hashes) {
@@ -547,12 +544,12 @@ class FullNode extends Node {
   /**
    * Retrieve transactions pertaining to an
    * address from the mempool or chain database.
-   * @param {Address} addrs
+   * @param {Address} addr
    * @returns {Promise} - Returns {@link TX}[].
    */
 
-  async getTXByAddress(addrs) {
+  async getTXByAddress(addr) {
-    const mtxs = await this.getMetaByAddress(addrs);
+    const mtxs = await this.getMetaByAddress(addr);
     const out = [];
 
     for (const mtx of mtxs)

test/indexer-test.js

@@ -60,7 +60,7 @@ const addrindexer = new AddrIndexer({
 describe('Indexer', function() {
   this.timeout(45000);
 
-  it('should open indexer', async () => {
+  before(async () => {
     await blocks.open();
     await chain.open();
     await miner.open();
@@ -68,62 +68,211 @@ describe('Indexer', function() {
     await addrindexer.open();
   });
 
-  it('should index 10 blocks', async () => {
+  after(async () => {
-    miner.addresses.length = 0;
+    await blocks.close();
-    miner.addAddress(wallet.getReceive());
+    await chain.close();
-    for (let i = 0; i < 10; i++) {
+    await miner.close();
-      const block = await cpu.mineBlock();
+    await txindexer.close();
-      assert(block);
+    await addrindexer.close();
-      assert(await chain.add(block));
-    }
-
-    assert.strictEqual(chain.height, 10);
-    assert.strictEqual(txindexer.state.startHeight, 10);
-    assert.strictEqual(addrindexer.state.startHeight, 10);
-
-    const coins =
-      await addrindexer.getCoinsByAddress(miner.getAddress());
-    assert.strictEqual(coins.length, 10);
-
-    for (const coin of coins) {
-      const meta = await txindexer.getMeta(coin.hash);
-      assert.bufferEqual(meta.tx.hash(), coin.hash);
-    }
   });
 
-  it('should rescan and reindex 10 missed blocks', async () => {
+  describe('index 10 blocks', function() {
-    for (let i = 0; i < 10; i++) {
+    before(async () => {
-      const block = await cpu.mineBlock();
+      miner.addresses.length = 0;
-      assert(block);
+      miner.addAddress(wallet.getReceive());
-      assert(await chain.add(block));
-    }
 
-    assert.strictEqual(chain.height, 20);
+      for (let i = 0; i < 10; i++) {
-    assert.strictEqual(txindexer.state.startHeight, 20);
+        const block = await cpu.mineBlock();
-    assert.strictEqual(addrindexer.state.startHeight, 20);
+        assert(block);
+        assert(await chain.add(block));
+      }
 
-    const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
+      assert.strictEqual(chain.height, 10);
-    assert.strictEqual(coins.length, 20);
+      assert.strictEqual(txindexer.state.startHeight, 10);
+      assert.strictEqual(addrindexer.state.startHeight, 10);
+    });
 
-    for (const coin of coins) {
+    it('should get coins by address', async () => {
-      const meta = await txindexer.getMeta(coin.hash);
+      const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
-      assert.bufferEqual(meta.tx.hash(), coin.hash);
+      assert.strictEqual(coins.length, 10);
-    }
+    });
+
+    it('should get txs by address', async () => {
+      const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
+      assert.strictEqual(hashes.length, 10);
+    });
+
+    it('should get txs for coins by address', async () => {
+      const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
+      assert.strictEqual(coins.length, 10);
+
+      for (const coin of coins) {
+        const meta = await txindexer.getMeta(coin.hash);
+        assert.bufferEqual(meta.tx.hash(), coin.hash);
+      }
+    });
+
+    it('should coins by address (limit)', async () => {
+      const addr = miner.getAddress();
+      const coins = await addrindexer.getCoinsByAddress(addr, {limit: 1});
+      assert.strictEqual(coins.length, 1);
+    });
+
+    it('should coins by address (reverse)', async () => {
+      const addr = miner.getAddress();
+      const coins = await addrindexer.getCoinsByAddress(
+        addr, {reverse: false});
+
+      assert.strictEqual(coins.length, 10);
+
+      const reversed = await addrindexer.getCoinsByAddress(
+        addr, {reverse: true});
+
+      assert.strictEqual(reversed.length, 10);
+
+      for (let i = 0; i < 10; i++)
+        assert.deepEqual(coins[i], reversed[9 - i]);
+    });
+
+    it('should get txs by address (limit)', async () => {
+      const addr = miner.getAddress();
+      const hashes = await addrindexer.getHashesByAddress(addr, {limit: 1});
+      assert.strictEqual(hashes.length, 1);
+    });
+
+    it('should get txs by address (reverse)', async () => {
+      const addr = miner.getAddress();
+      const hashes = await addrindexer.getHashesByAddress(
+        addr, {reverse: false});
+
+      assert.strictEqual(hashes.length, 10);
+
+      const reversed = await addrindexer.getHashesByAddress(
+        addr, {reverse: true});
+
+      assert.strictEqual(reversed.length, 10);
+
+      for (let i = 0; i < 10; i++)
+        assert.deepEqual(hashes[i], reversed[9 - i]);
+    });
+
+    it('should coins by address after txid and index', async () => {
+      const addr = miner.getAddress();
+      const coins = await addrindexer.getCoinsByAddress(addr, {limit: 5});
+
+      assert.strictEqual(coins.length, 5);
+
+      const txid = coins[4].hash;
+      const index = coins[4].index;
+
+      const next = await addrindexer.getCoinsByAddressAfter(
+        addr, {txid: txid, index: index, limit: 5});
+
+      assert.strictEqual(next.length, 5);
+
+      const all = await addrindexer.getCoinsByAddress(addr);
+      assert.strictEqual(all.length, 10);
+
+      assert.deepEqual(coins.concat(next), all);
+    });
+
+    it('should coins by address after txid and index (reverse)', async () => {
+      const addr = miner.getAddress();
+      const coins = await addrindexer.getCoinsByAddress(
+        addr, {limit: 5, reverse: true});
+
+      assert.strictEqual(coins.length, 5);
+
+      const txid = coins[4].hash;
+      const index = coins[4].index;
+
+      const next = await addrindexer.getCoinsByAddressAfter(
+        addr, {txid: txid, index: index, limit: 5, reverse: true});
+
+      assert.strictEqual(next.length, 5);
+
+      const all = await addrindexer.getCoinsByAddress(addr, {reverse: true});
+      assert.strictEqual(all.length, 10);
+
+      assert.deepEqual(coins.concat(next), all);
+    });
+
+    it('should txs by address after txid', async () => {
+      const addr = miner.getAddress();
+      const hashes = await addrindexer.getHashesByAddress(addr, {limit: 5});
+
+      assert.strictEqual(hashes.length, 5);
+
+      const txid = hashes[4];
+
+      const next = await addrindexer.getHashesByAddressAfter(
+        addr, {txid: txid, limit: 5});
+
+      assert.strictEqual(next.length, 5);
+
+      const all = await addrindexer.getHashesByAddress(addr);
+      assert.strictEqual(all.length, 10);
+
+      assert.deepEqual(hashes.concat(next), all);
+    });
+
+    it('should txs by address after txid (reverse)', async () => {
+      const addr = miner.getAddress();
+      const hashes = await addrindexer.getHashesByAddress(
+        addr, {limit: 5, reverse: true});
+
+      assert.strictEqual(hashes.length, 5);
+
+      const txid = hashes[4];
+
+      const next = await addrindexer.getHashesByAddressAfter(
+        addr, {txid: txid, limit: 5, reverse: true});
+
+      assert.strictEqual(next.length, 5);
+
+      const all = await addrindexer.getHashesByAddress(
+        addr, {reverse: true});
+
+      assert.strictEqual(all.length, 10);
+
+      assert.deepEqual(hashes.concat(next), all);
+    });
   });
 
-  it('should handle indexing a reorg', async () => {
+  describe('rescan and reorg', function() {
-    await reorg(chain, cpu, 10);
+    it('should rescan and reindex 10 missed blocks', async () => {
+      for (let i = 0; i < 10; i++) {
+        const block = await cpu.mineBlock();
+        assert(block);
+        assert(await chain.add(block));
+      }
 
-    assert.strictEqual(txindexer.state.startHeight, 31);
+      assert.strictEqual(chain.height, 20);
-    assert.strictEqual(addrindexer.state.startHeight, 31);
+      assert.strictEqual(txindexer.state.startHeight, 20);
+      assert.strictEqual(addrindexer.state.startHeight, 20);
 
-    const coins =
+      const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
-      await addrindexer.getCoinsByAddress(miner.getAddress());
+      assert.strictEqual(coins.length, 20);
-    assert.strictEqual(coins.length, 31);
 
-    for (const coin of coins) {
+      for (const coin of coins) {
-      const meta = await txindexer.getMeta(coin.hash);
+        const meta = await txindexer.getMeta(coin.hash);
-      assert.bufferEqual(meta.tx.hash(), coin.hash);
+        assert.bufferEqual(meta.tx.hash(), coin.hash);
-    }
+      }
+    });
+
+    it('should handle indexing a reorg', async () => {
+      await reorg(chain, cpu, 10);
+
+      assert.strictEqual(txindexer.state.startHeight, 31);
+      assert.strictEqual(addrindexer.state.startHeight, 31);
+
+      const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
+      assert.strictEqual(coins.length, 31);
+
+      for (const coin of coins) {
+        const meta = await txindexer.getMeta(coin.hash);
+        assert.bufferEqual(meta.tx.hash(), coin.hash);
+      }
+    });
   });
 });