diff --git a/lib/indexer/addrindexer.js b/lib/indexer/addrindexer.js index 765398f8..d9375f0b 100644 --- a/lib/indexer/addrindexer.js +++ b/lib/indexer/addrindexer.js @@ -8,6 +8,7 @@ const assert = require('assert'); const bdb = require('bdb'); +const bio = require('bufio'); const {BufferSet} = require('buffer-map'); const layout = require('./layout'); const Address = require('../primitives/address'); @@ -15,15 +16,93 @@ const Indexer = require('./indexer'); /* * AddrIndexer Database Layout: - * T[addr-hash][hash] -> dummy (tx by address) - * C[addr-hash][hash][index] -> dummy (coin by address) + * T[addr-hash][height][tx-index][hash] -> dummy (tx by address) + * C[addr-hash][height][tx-index][hash][coin-index] -> dummy (coin by address) + * x[addr-hash][hash] -> height and tx-index for tx + * y[addr-hash][hash][index] -> height, tx-index and coin-index for coin */ Object.assign(layout, { - T: bdb.key('T', ['hash', 'hash256']), - C: bdb.key('C', ['hash', 'hash256', 'uint32']) + T: bdb.key('T', ['hash', 'uint32', 'uint32', 'hash256']), + C: bdb.key('C', ['hash', 'uint32', 'uint32', 'hash256', 'uint32']), + x: bdb.key('x', ['hash', 'hash256']), + y: bdb.key('y', ['hash', 'hash256', 'uint32']) }); +/** + * Count + */ + +class Count { + /** + * Create count record. + * @constructor + * @param {Number} height + * @param {Number} index + */ + + constructor(height, index, coin) { + this.height = height >= 0 ? height : 0; + this.index = index >= 0 ? index : 0; + this.coin = coin >= 0 ? coin : -1; + + assert((this.height >>> 0) === this.height); + assert((this.index >>> 0) === this.index); + + if (coin) + assert((this.coin >>> 0) === this.coin); + } + + /** + * Serialize. + * @returns {Buffer} + */ + + toRaw() { + let len = 8; + if (this.coin >= 0) + len += 4; + + const bw = bio.write(len); + + bw.writeU32(this.height); + bw.writeU32(this.index); + + if (this.coin >= 0) + bw.writeU32(this.coin); + + return bw.render(); + } + + /** + * Deserialize. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.height = br.readU32(); + this.index = br.readU32(); + + if (br.left() >= 4) + this.coin = br.readU32(); + + return this; + } + + /** + * Instantiate a count from a buffer. + * @param {Buffer} data + * @returns {Count} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } +} + /** * AddrIndexer * @alias module:indexer.AddrIndexer @@ -41,6 +120,8 @@ class AddrIndexer extends Indexer { super('addr', options); this.db = bdb.create(this.options); + this.maxTxs = options.maxTxs || 100; + this.maxCoins = options.maxCoins || 500; } /** @@ -53,12 +134,18 @@ class AddrIndexer extends Indexer { async indexBlock(entry, block, view) { const b = this.db.batch(); + const height = entry.height; for (let i = 0; i < block.txs.length; i++) { const tx = block.txs[i]; const hash = tx.hash(); - for (const addr of tx.getHashes(view)) - b.put(layout.T.encode(addr, hash), null); + + for (const addr of tx.getHashes(view)) { + const count = new Count(height, i); + + b.put(layout.T.encode(addr, height, i, hash), null); + b.put(layout.x.encode(addr, hash), count.toRaw()); + } if (!tx.isCoinbase()) { for (const {prevout} of tx.inputs) { @@ -71,18 +158,22 @@ class AddrIndexer extends Indexer { if (!addr) continue; - b.del(layout.C.encode(addr, hash, index)); + b.del(layout.C.encode(addr, height, i, hash, index)); + b.del(layout.y.encode(addr, hash, index)); } } - for (let i = 0; i < tx.outputs.length; i++) { - const output = tx.outputs[i]; + for (let j = 0; j < tx.outputs.length; j++) { + const output = tx.outputs[j]; const addr = output.getHash(); if (!addr) continue; - b.put(layout.C.encode(addr, hash, i), null); + const count = new Count(height, i, j); + + b.put(layout.C.encode(addr, height, i, hash, j), null); + b.put(layout.y.encode(addr, hash, j), count.toRaw()); } } @@ -99,11 +190,16 @@ class AddrIndexer extends Indexer { async unindexBlock(entry, block, view) { const b = this.db.batch(); + const height = entry.height; + for (let i = 0; i < block.txs.length; i++) { const tx = block.txs[i]; const hash = tx.hash(); - for (const addr of tx.getHashes(view)) - b.del(layout.T.encode(addr, hash)); + + for (const addr of tx.getHashes(view)) { + b.del(layout.T.encode(addr, height, i, hash)); + b.del(layout.x.encode(addr, hash)); + } if (!tx.isCoinbase()) { for (const {prevout} of tx.inputs) { @@ -116,18 +212,22 @@ class AddrIndexer extends Indexer { if (!addr) continue; - b.put(layout.C.encode(addr, hash, index), null); + const count = new Count(height, i); + + b.put(layout.C.encode(addr, height, i, hash, index), null); + b.put(layout.y.encode(addr, hash, index), count.toRaw()); } } - for (let i = 0; i < tx.outputs.length; i++) { - const output = tx.outputs[i]; + for (let j = 0; j < tx.outputs.length; j++) { + const output = tx.outputs[j]; const addr = output.getHash(); if (!addr) continue; - b.del(layout.C.encode(addr, hash, i)); + b.del(layout.C.encode(addr, height, i, hash, j)); + b.del(layout.y.encode(addr, hash, j)); } } @@ -136,33 +236,103 @@ class AddrIndexer extends Indexer { /** * Get all coins pertinent to an address. - * @param {Address[]} addrs + * @param {Address} addr + * @param {Object} options + * @param {Boolean} options.reverse + * @param {Boolean} options.limit * @returns {Promise} - Returns {@link Coin}[]. */ - async getCoinsByAddress(addrs) { - if (!Array.isArray(addrs)) - addrs = [addrs]; - + async getCoinsByAddress(addr, options = {}) { const coins = []; - for (const addr of addrs) { - const hash = Address.getHash(addr); + const {reverse} = options; + let {limit} = options; - const keys = await this.db.keys({ - gte: layout.C.min(hash), - lte: layout.C.max(hash), - parse: (key) => { - const [, txid, index] = layout.C.decode(key); - return [txid, index]; - } - }); + if (!limit) + limit = this.maxCoins; - for (const [hash, index] of keys) { - const coin = await this.chain.getCoin(hash, index); - assert(coin); - coins.push(coin); + if (limit > this.maxCoins) + throw new Error('Limit above max of ${this.maxCoins}.'); + + const hash = Address.getHash(addr); + + const keys = await this.db.keys({ + gte: layout.C.min(hash), + lte: layout.C.max(hash), + limit, + reverse, + parse: (key) => { + const [,,, txid, index] = layout.C.decode(key); + return [txid, index]; } + }); + + for (const [hash, index] of keys) { + const coin = await this.chain.getCoin(hash, index); + assert(coin); + coins.push(coin); + } + + return coins; + } + + /** + * Get all coins pertinent to an address after a + * specific txid and output/coin index. + * @param {Address} addr + * @param {Object} options + * @param {Buffer} options.txid + * @param {Number} options.index + * @param {Boolean} options.limit + * @param {Boolean} options.reverse + * @returns {Promise} - Returns {@link Coin}[]. + */ + + async getCoinsByAddressAfter(addr, options = {}) { + const coins = []; + + const {txid, index, reverse} = options; + let {limit} = options; + + if (!limit) + limit = this.maxCoins; + + if (limit > this.maxCoins) + throw new Error('Limit above max of ${this.maxCoins}.'); + + const hash = Address.getHash(addr); + + const raw = await this.db.get(layout.y.encode(hash, txid, index)); + + if (!raw) + return coins; + + const count = Count.fromRaw(raw); + + const opts = { + limit, + reverse, + parse: (key) => { + const [,,, txid, index] = layout.C.decode(key); + return [txid, index]; + } + }; + + if (!reverse) { + opts.gt = layout.C.min(hash, count.height, count.index, txid, count.coin); + opts.lte = layout.C.max(hash); + } else { + opts.gte = layout.C.min(hash); + opts.lt = layout.C.max(hash, count.height, count.index, txid, count.coin); + } + + const keys = await this.db.keys(opts); + + for (const [hash, index] of keys) { + const coin = await this.chain.getCoin(hash, index); + assert(coin); + coins.push(coin); } return coins; @@ -170,26 +340,93 @@ class AddrIndexer extends Indexer { /** * Get all transaction hashes to an address. - * @param {Address[]} addrs + * @param {Address} addr + * @param {Object} options + * @param {Boolean} options.limit + * @param {Boolean} options.reverse * @returns {Promise} - Returns {@link Hash}[]. */ - async getHashesByAddress(addrs) { + async getHashesByAddress(addr, options = {}) { const set = new BufferSet(); - for (const addr of addrs) { - const hash = Address.getHash(addr); + const {reverse} = options; + let {limit} = options; - await this.db.keys({ - gte: layout.T.min(hash), - lte: layout.T.max(hash), - parse: (key) => { - const [, txid] = layout.T.decode(key); - set.add(txid); - } - }); + if (!limit) + limit = this.maxTxs; + + if (limit > this.maxTxs) + throw new Error('Limit above max of ${this.maxTxs}.'); + + const hash = Address.getHash(addr); + + await this.db.keys({ + gte: layout.T.min(hash), + lte: layout.T.max(hash), + limit, + reverse, + parse: (key) => { + const [,,, txid] = layout.T.decode(key); + set.add(txid); + } + }); + + return set.toArray(); + } + + /** + * Get all transaction hashes to an address after + * a specific txid. + * @param {Address} addr + * @param {Object} options + * @param {Buffer} options.txid + * @param {Boolean} options.limit + * @param {Boolean} options.reverse + * @returns {Promise} - Returns {@link Hash}[]. + */ + + async getHashesByAddressAfter(addr, options = {}) { + const set = new BufferSet(); + + const hash = Address.getHash(addr); + + const {txid, reverse} = options; + let {limit} = options; + + if (!limit) + limit = this.maxTxs; + + if (limit > this.maxTxs) + throw new Error('Limit above max of ${this.maxTxs}.'); + + const raw = await this.db.get(layout.x.encode(hash, txid)); + + if (!raw) + return []; + + const count = Count.fromRaw(raw); + const {height, index} = count; + + const opts = { + limit, + reverse, + parse: (key) => { + const [,,, txid] = layout.T.decode(key); + set.add(txid); + } + }; + + if (!reverse) { + opts.gt = layout.T.min(hash, height, index, txid); + opts.lte = layout.T.max(hash); + } else { + opts.gte = layout.T.min(hash); + opts.lt = layout.T.max(hash, height, index, txid); } + await this.db.keys(opts); + return set.toArray(); } } diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index b7969b67..803e1fa3 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -454,17 +454,17 @@ class FullNode extends Node { /** * Get coins that pertain to an address from the mempool or chain database. * Takes into account spent coins in the mempool. - * @param {Address} addrs + * @param {Address} addr * @returns {Promise} - Returns {@link Coin}[]. */ - async getCoinsByAddress(addrs) { - const mempool = this.mempool.getCoinsByAddress(addrs); + async getCoinsByAddress(addr) { + const mempool = this.mempool.getCoinsByAddress(addr); if (!this.addrindex) return mempool; - const index = await this.addrindex.getCoinsByAddress(addrs); + const index = await this.addrindex.getCoinsByAddress(addr); const out = []; for (const coin of index) { @@ -485,18 +485,15 @@ class FullNode extends Node { /** * Retrieve transactions pertaining to an * address from the mempool or chain database. - * @param {Address} addrs + * @param {Address} addr * @returns {Promise} - Returns {@link TXMeta}[]. */ - async getMetaByAddress(addrs) { - const mempool = this.mempool.getMetaByAddress(addrs); + async getMetaByAddress(addr) { + const mempool = this.mempool.getMetaByAddress(addr); if (this.txindex && this.addrindex) { - if (!Array.isArray(addrs)) - addrs = [addrs]; - - const hashes = await this.addrindex.getHashesByAddress(addrs); + const hashes = await this.addrindex.getHashesByAddress(addr); const mtxs = []; for (const hash of hashes) { @@ -547,12 +544,12 @@ class FullNode extends Node { /** * Retrieve transactions pertaining to an * address from the mempool or chain database. - * @param {Address} addrs + * @param {Address} addr * @returns {Promise} - Returns {@link TX}[]. */ - async getTXByAddress(addrs) { - const mtxs = await this.getMetaByAddress(addrs); + async getTXByAddress(addr) { + const mtxs = await this.getMetaByAddress(addr); const out = []; for (const mtx of mtxs) diff --git a/test/indexer-test.js b/test/indexer-test.js index 3f476da7..b3e49561 100644 --- a/test/indexer-test.js +++ b/test/indexer-test.js @@ -60,7 +60,7 @@ const addrindexer = new AddrIndexer({ describe('Indexer', function() { this.timeout(45000); - it('should open indexer', async () => { + before(async () => { await blocks.open(); await chain.open(); await miner.open(); @@ -68,62 +68,211 @@ describe('Indexer', function() { await addrindexer.open(); }); - it('should index 10 blocks', async () => { - miner.addresses.length = 0; - miner.addAddress(wallet.getReceive()); - for (let i = 0; i < 10; i++) { - const block = await cpu.mineBlock(); - assert(block); - assert(await chain.add(block)); - } - - assert.strictEqual(chain.height, 10); - assert.strictEqual(txindexer.state.startHeight, 10); - assert.strictEqual(addrindexer.state.startHeight, 10); - - const coins = - await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 10); - - for (const coin of coins) { - const meta = await txindexer.getMeta(coin.hash); - assert.bufferEqual(meta.tx.hash(), coin.hash); - } + after(async () => { + await blocks.close(); + await chain.close(); + await miner.close(); + await txindexer.close(); + await addrindexer.close(); }); - it('should rescan and reindex 10 missed blocks', async () => { - for (let i = 0; i < 10; i++) { - const block = await cpu.mineBlock(); - assert(block); - assert(await chain.add(block)); - } + describe('index 10 blocks', function() { + before(async () => { + miner.addresses.length = 0; + miner.addAddress(wallet.getReceive()); - assert.strictEqual(chain.height, 20); - assert.strictEqual(txindexer.state.startHeight, 20); - assert.strictEqual(addrindexer.state.startHeight, 20); + for (let i = 0; i < 10; i++) { + const block = await cpu.mineBlock(); + assert(block); + assert(await chain.add(block)); + } - const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 20); + assert.strictEqual(chain.height, 10); + assert.strictEqual(txindexer.state.startHeight, 10); + assert.strictEqual(addrindexer.state.startHeight, 10); + }); - for (const coin of coins) { - const meta = await txindexer.getMeta(coin.hash); - assert.bufferEqual(meta.tx.hash(), coin.hash); - } + it('should get coins by address', async () => { + const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 10); + }); + + it('should get txs by address', async () => { + const hashes = await addrindexer.getHashesByAddress(miner.getAddress()); + assert.strictEqual(hashes.length, 10); + }); + + it('should get txs for coins by address', async () => { + const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 10); + + for (const coin of coins) { + const meta = await txindexer.getMeta(coin.hash); + assert.bufferEqual(meta.tx.hash(), coin.hash); + } + }); + + it('should coins by address (limit)', async () => { + const addr = miner.getAddress(); + const coins = await addrindexer.getCoinsByAddress(addr, {limit: 1}); + assert.strictEqual(coins.length, 1); + }); + + it('should coins by address (reverse)', async () => { + const addr = miner.getAddress(); + const coins = await addrindexer.getCoinsByAddress( + addr, {reverse: false}); + + assert.strictEqual(coins.length, 10); + + const reversed = await addrindexer.getCoinsByAddress( + addr, {reverse: true}); + + assert.strictEqual(reversed.length, 10); + + for (let i = 0; i < 10; i++) + assert.deepEqual(coins[i], reversed[9 - i]); + }); + + it('should get txs by address (limit)', async () => { + const addr = miner.getAddress(); + const hashes = await addrindexer.getHashesByAddress(addr, {limit: 1}); + assert.strictEqual(hashes.length, 1); + }); + + it('should get txs by address (reverse)', async () => { + const addr = miner.getAddress(); + const hashes = await addrindexer.getHashesByAddress( + addr, {reverse: false}); + + assert.strictEqual(hashes.length, 10); + + const reversed = await addrindexer.getHashesByAddress( + addr, {reverse: true}); + + assert.strictEqual(reversed.length, 10); + + for (let i = 0; i < 10; i++) + assert.deepEqual(hashes[i], reversed[9 - i]); + }); + + it('should coins by address after txid and index', async () => { + const addr = miner.getAddress(); + const coins = await addrindexer.getCoinsByAddress(addr, {limit: 5}); + + assert.strictEqual(coins.length, 5); + + const txid = coins[4].hash; + const index = coins[4].index; + + const next = await addrindexer.getCoinsByAddressAfter( + addr, {txid: txid, index: index, limit: 5}); + + assert.strictEqual(next.length, 5); + + const all = await addrindexer.getCoinsByAddress(addr); + assert.strictEqual(all.length, 10); + + assert.deepEqual(coins.concat(next), all); + }); + + it('should coins by address after txid and index (reverse)', async () => { + const addr = miner.getAddress(); + const coins = await addrindexer.getCoinsByAddress( + addr, {limit: 5, reverse: true}); + + assert.strictEqual(coins.length, 5); + + const txid = coins[4].hash; + const index = coins[4].index; + + const next = await addrindexer.getCoinsByAddressAfter( + addr, {txid: txid, index: index, limit: 5, reverse: true}); + + assert.strictEqual(next.length, 5); + + const all = await addrindexer.getCoinsByAddress(addr, {reverse: true}); + assert.strictEqual(all.length, 10); + + assert.deepEqual(coins.concat(next), all); + }); + + it('should txs by address after txid', async () => { + const addr = miner.getAddress(); + const hashes = await addrindexer.getHashesByAddress(addr, {limit: 5}); + + assert.strictEqual(hashes.length, 5); + + const txid = hashes[4]; + + const next = await addrindexer.getHashesByAddressAfter( + addr, {txid: txid, limit: 5}); + + assert.strictEqual(next.length, 5); + + const all = await addrindexer.getHashesByAddress(addr); + assert.strictEqual(all.length, 10); + + assert.deepEqual(hashes.concat(next), all); + }); + + it('should txs by address after txid (reverse)', async () => { + const addr = miner.getAddress(); + const hashes = await addrindexer.getHashesByAddress( + addr, {limit: 5, reverse: true}); + + assert.strictEqual(hashes.length, 5); + + const txid = hashes[4]; + + const next = await addrindexer.getHashesByAddressAfter( + addr, {txid: txid, limit: 5, reverse: true}); + + assert.strictEqual(next.length, 5); + + const all = await addrindexer.getHashesByAddress( + addr, {reverse: true}); + + assert.strictEqual(all.length, 10); + + assert.deepEqual(hashes.concat(next), all); + }); }); - it('should handle indexing a reorg', async () => { - await reorg(chain, cpu, 10); + describe('rescan and reorg', function() { + it('should rescan and reindex 10 missed blocks', async () => { + for (let i = 0; i < 10; i++) { + const block = await cpu.mineBlock(); + assert(block); + assert(await chain.add(block)); + } - assert.strictEqual(txindexer.state.startHeight, 31); - assert.strictEqual(addrindexer.state.startHeight, 31); + assert.strictEqual(chain.height, 20); + assert.strictEqual(txindexer.state.startHeight, 20); + assert.strictEqual(addrindexer.state.startHeight, 20); - const coins = - await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 31); + const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 20); - for (const coin of coins) { - const meta = await txindexer.getMeta(coin.hash); - assert.bufferEqual(meta.tx.hash(), coin.hash); - } + for (const coin of coins) { + const meta = await txindexer.getMeta(coin.hash); + assert.bufferEqual(meta.tx.hash(), coin.hash); + } + }); + + it('should handle indexing a reorg', async () => { + await reorg(chain, cpu, 10); + + assert.strictEqual(txindexer.state.startHeight, 31); + assert.strictEqual(addrindexer.state.startHeight, 31); + + const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); + assert.strictEqual(coins.length, 31); + + for (const coin of coins) { + const meta = await txindexer.getMeta(coin.hash); + assert.bufferEqual(meta.tx.hash(), coin.hash); + } + }); }); });