From 7dc55c9c3cc4690e4c050f10d6802d3bda0fdea4 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Mar 2019 15:30:12 -0700 Subject: [PATCH] indexer: remove address coin index --- lib/indexer/addrindexer.js | 234 +++++-------------------------------- lib/node/fullnode.js | 31 ----- lib/node/http.js | 54 --------- test/indexer-test.js | 98 ++-------------- test/node-test.js | 32 ----- 5 files changed, 42 insertions(+), 407 deletions(-) diff --git a/lib/indexer/addrindexer.js b/lib/indexer/addrindexer.js index d9375f0b..a9c5d593 100644 --- a/lib/indexer/addrindexer.js +++ b/lib/indexer/addrindexer.js @@ -16,17 +16,24 @@ const Indexer = require('./indexer'); /* * AddrIndexer Database Layout: - * T[addr-hash][height][tx-index][hash] -> dummy (tx by address) - * C[addr-hash][height][tx-index][hash][coin-index] -> dummy (coin by address) - * x[addr-hash][hash] -> height and tx-index for tx - * y[addr-hash][hash][index] -> height, tx-index and coin-index for coin -*/ + * A[addr-hash][height][index][hash] -> dummy (tx by address, height and index) + * a[addr-hash][hash] -> (tx height and index by address and tx hash) + * + * The database layout is organized so that transactions are sorted in + * the same order as the blocks (e.g. chronological order) using the block + * height and transaction index. This provides the ability to query for + * sets of transactions within that order. For a wallet that would like to + * synchronize or rescan, this could be a query for all of the latest + * transactions, but not for earlier transactions that are already known. + * Furthermore, to be able to query for all transactions in multiple sets + * without reference to height and index, there is a mapping from address + * and tx hash to the height and index as an entry point to the + * ordered transactions. + */ Object.assign(layout, { - T: bdb.key('T', ['hash', 'uint32', 'uint32', 'hash256']), - C: bdb.key('C', ['hash', 'uint32', 'uint32', 'hash256', 'uint32']), - x: bdb.key('x', ['hash', 'hash256']), - y: bdb.key('y', ['hash', 'hash256', 'uint32']) + A: bdb.key('A', ['hash', 'uint32', 'uint32', 'hash256']), + a: bdb.key('a', ['hash', 'hash256']) }); /** @@ -42,15 +49,11 @@ class Count { */ constructor(height, index, coin) { - this.height = height >= 0 ? height : 0; - this.index = index >= 0 ? index : 0; - this.coin = coin >= 0 ? coin : -1; + this.height = height || 0; + this.index = index || 0; assert((this.height >>> 0) === this.height); assert((this.index >>> 0) === this.index); - - if (coin) - assert((this.coin >>> 0) === this.coin); } /** @@ -59,18 +62,11 @@ class Count { */ toRaw() { - let len = 8; - if (this.coin >= 0) - len += 4; - - const bw = bio.write(len); + const bw = bio.write(8); bw.writeU32(this.height); bw.writeU32(this.index); - if (this.coin >= 0) - bw.writeU32(this.coin); - return bw.render(); } @@ -86,9 +82,6 @@ class Count { this.height = br.readU32(); this.index = br.readU32(); - if (br.left() >= 4) - this.coin = br.readU32(); - return this; } @@ -121,7 +114,6 @@ class AddrIndexer extends Indexer { this.db = bdb.create(this.options); this.maxTxs = options.maxTxs || 100; - this.maxCoins = options.maxCoins || 500; } /** @@ -143,37 +135,8 @@ class AddrIndexer extends Indexer { for (const addr of tx.getHashes(view)) { const count = new Count(height, i); - b.put(layout.T.encode(addr, height, i, hash), null); - b.put(layout.x.encode(addr, hash), count.toRaw()); - } - - if (!tx.isCoinbase()) { - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - const coin = view.getOutput(prevout); - assert(coin); - - const addr = coin.getHash(); - - if (!addr) - continue; - - b.del(layout.C.encode(addr, height, i, hash, index)); - b.del(layout.y.encode(addr, hash, index)); - } - } - - for (let j = 0; j < tx.outputs.length; j++) { - const output = tx.outputs[j]; - const addr = output.getHash(); - - if (!addr) - continue; - - const count = new Count(height, i, j); - - b.put(layout.C.encode(addr, height, i, hash, j), null); - b.put(layout.y.encode(addr, hash, j), count.toRaw()); + b.put(layout.A.encode(addr, height, i, hash), null); + b.put(layout.a.encode(addr, hash), count.toRaw()); } } @@ -197,147 +160,14 @@ class AddrIndexer extends Indexer { const hash = tx.hash(); for (const addr of tx.getHashes(view)) { - b.del(layout.T.encode(addr, height, i, hash)); - b.del(layout.x.encode(addr, hash)); - } - - if (!tx.isCoinbase()) { - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - const coin = view.getOutput(prevout); - assert(coin); - - const addr = coin.getHash(); - - if (!addr) - continue; - - const count = new Count(height, i); - - b.put(layout.C.encode(addr, height, i, hash, index), null); - b.put(layout.y.encode(addr, hash, index), count.toRaw()); - } - } - - for (let j = 0; j < tx.outputs.length; j++) { - const output = tx.outputs[j]; - const addr = output.getHash(); - - if (!addr) - continue; - - b.del(layout.C.encode(addr, height, i, hash, j)); - b.del(layout.y.encode(addr, hash, j)); + b.del(layout.A.encode(addr, height, i, hash)); + b.del(layout.a.encode(addr, hash)); } } return b.write(); } - /** - * Get all coins pertinent to an address. - * @param {Address} addr - * @param {Object} options - * @param {Boolean} options.reverse - * @param {Boolean} options.limit - * @returns {Promise} - Returns {@link Coin}[]. - */ - - async getCoinsByAddress(addr, options = {}) { - const coins = []; - - const {reverse} = options; - let {limit} = options; - - if (!limit) - limit = this.maxCoins; - - if (limit > this.maxCoins) - throw new Error('Limit above max of ${this.maxCoins}.'); - - const hash = Address.getHash(addr); - - const keys = await this.db.keys({ - gte: layout.C.min(hash), - lte: layout.C.max(hash), - limit, - reverse, - parse: (key) => { - const [,,, txid, index] = layout.C.decode(key); - return [txid, index]; - } - }); - - for (const [hash, index] of keys) { - const coin = await this.chain.getCoin(hash, index); - assert(coin); - coins.push(coin); - } - - return coins; - } - - /** - * Get all coins pertinent to an address after a - * specific txid and output/coin index. - * @param {Address} addr - * @param {Object} options - * @param {Buffer} options.txid - * @param {Number} options.index - * @param {Boolean} options.limit - * @param {Boolean} options.reverse - * @returns {Promise} - Returns {@link Coin}[]. - */ - - async getCoinsByAddressAfter(addr, options = {}) { - const coins = []; - - const {txid, index, reverse} = options; - let {limit} = options; - - if (!limit) - limit = this.maxCoins; - - if (limit > this.maxCoins) - throw new Error('Limit above max of ${this.maxCoins}.'); - - const hash = Address.getHash(addr); - - const raw = await this.db.get(layout.y.encode(hash, txid, index)); - - if (!raw) - return coins; - - const count = Count.fromRaw(raw); - - const opts = { - limit, - reverse, - parse: (key) => { - const [,,, txid, index] = layout.C.decode(key); - return [txid, index]; - } - }; - - if (!reverse) { - opts.gt = layout.C.min(hash, count.height, count.index, txid, count.coin); - opts.lte = layout.C.max(hash); - } else { - opts.gte = layout.C.min(hash); - opts.lt = layout.C.max(hash, count.height, count.index, txid, count.coin); - } - - const keys = await this.db.keys(opts); - - for (const [hash, index] of keys) { - const coin = await this.chain.getCoin(hash, index); - assert(coin); - coins.push(coin); - } - - return coins; - } - /** * Get all transaction hashes to an address. * @param {Address} addr @@ -362,12 +192,12 @@ class AddrIndexer extends Indexer { const hash = Address.getHash(addr); await this.db.keys({ - gte: layout.T.min(hash), - lte: layout.T.max(hash), + gte: layout.A.min(hash), + lte: layout.A.max(hash), limit, reverse, parse: (key) => { - const [,,, txid] = layout.T.decode(key); + const [,,, txid] = layout.A.decode(key); set.add(txid); } }); @@ -400,7 +230,7 @@ class AddrIndexer extends Indexer { if (limit > this.maxTxs) throw new Error('Limit above max of ${this.maxTxs}.'); - const raw = await this.db.get(layout.x.encode(hash, txid)); + const raw = await this.db.get(layout.a.encode(hash, txid)); if (!raw) return []; @@ -412,17 +242,17 @@ class AddrIndexer extends Indexer { limit, reverse, parse: (key) => { - const [,,, txid] = layout.T.decode(key); + const [,,, txid] = layout.A.decode(key); set.add(txid); } }; if (!reverse) { - opts.gt = layout.T.min(hash, height, index, txid); - opts.lte = layout.T.max(hash); + opts.gt = layout.A.min(hash, height, index, txid); + opts.lte = layout.A.max(hash); } else { - opts.gte = layout.T.min(hash); - opts.lt = layout.T.max(hash, height, index, txid); + opts.gte = layout.A.min(hash); + opts.lt = layout.A.max(hash, height, index, txid); } await this.db.keys(opts); diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 803e1fa3..4e6747d4 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -451,37 +451,6 @@ class FullNode extends Node { return this.chain.getCoin(hash, index); } - /** - * Get coins that pertain to an address from the mempool or chain database. - * Takes into account spent coins in the mempool. - * @param {Address} addr - * @returns {Promise} - Returns {@link Coin}[]. - */ - - async getCoinsByAddress(addr) { - const mempool = this.mempool.getCoinsByAddress(addr); - - if (!this.addrindex) - return mempool; - - const index = await this.addrindex.getCoinsByAddress(addr); - const out = []; - - for (const coin of index) { - const spent = this.mempool.isSpent(coin.hash, coin.index); - - if (spent) - continue; - - out.push(coin); - } - - for (const coin of mempool) - out.push(coin); - - return out; - } - /** * Retrieve transactions pertaining to an * address from the mempool or chain database. diff --git a/lib/node/http.js b/lib/node/http.js index 2c9b4dfa..40e80f32 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -149,24 +149,6 @@ class HTTP extends Server { }); }); - // UTXO by address - this.get('/coin/address/:address', async (req, res) => { - const valid = Validator.fromRequest(req); - const address = valid.str('address'); - - enforce(address, 'Address is required.'); - enforce(!this.chain.options.spv, 'Cannot get coins in SPV mode.'); - - const addr = Address.fromString(address, this.network); - const coins = await this.node.getCoinsByAddress(addr); - const result = []; - - for (const coin of coins) - result.push(coin.getJSON(this.network)); - - res.json(200, result); - }); - // UTXO by id this.get('/coin/:hash/:index', async (req, res) => { const valid = Validator.fromRequest(req); @@ -187,23 +169,6 @@ class HTTP extends Server { res.json(200, coin.getJSON(this.network)); }); - // Bulk read UTXOs - this.post('/coin/address', async (req, res) => { - const valid = Validator.fromRequest(req); - const address = valid.array('addresses'); - - enforce(address, 'Address is required.'); - enforce(!this.chain.options.spv, 'Cannot get coins in SPV mode.'); - - const coins = await this.node.getCoinsByAddress(address); - const result = []; - - for (const coin of coins) - result.push(coin.getJSON(this.network)); - - res.json(200, result); - }); - // TX by hash this.get('/tx/:hash', async (req, res) => { const valid = Validator.fromRequest(req); @@ -244,25 +209,6 @@ class HTTP extends Server { res.json(200, result); }); - // Bulk read TXs - this.post('/tx/address', async (req, res) => { - const valid = Validator.fromRequest(req); - const address = valid.array('addresses'); - - enforce(address, 'Address is required.'); - enforce(!this.chain.options.spv, 'Cannot get TX in SPV mode.'); - - const metas = await this.node.getMetaByAddress(address); - const result = []; - - for (const meta of metas) { - const view = await this.node.getMetaView(meta); - result.push(meta.getJSON(this.network, view, this.chain.height)); - } - - res.json(200, result); - }); - // Block by hash/height this.get('/block/:block', async (req, res) => { const valid = Validator.fromRequest(req); diff --git a/test/indexer-test.js b/test/indexer-test.js index b3e49561..085a7c7d 100644 --- a/test/indexer-test.js +++ b/test/indexer-test.js @@ -92,48 +92,11 @@ describe('Indexer', function() { assert.strictEqual(addrindexer.state.startHeight, 10); }); - it('should get coins by address', async () => { - const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 10); - }); - it('should get txs by address', async () => { const hashes = await addrindexer.getHashesByAddress(miner.getAddress()); assert.strictEqual(hashes.length, 10); }); - it('should get txs for coins by address', async () => { - const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 10); - - for (const coin of coins) { - const meta = await txindexer.getMeta(coin.hash); - assert.bufferEqual(meta.tx.hash(), coin.hash); - } - }); - - it('should coins by address (limit)', async () => { - const addr = miner.getAddress(); - const coins = await addrindexer.getCoinsByAddress(addr, {limit: 1}); - assert.strictEqual(coins.length, 1); - }); - - it('should coins by address (reverse)', async () => { - const addr = miner.getAddress(); - const coins = await addrindexer.getCoinsByAddress( - addr, {reverse: false}); - - assert.strictEqual(coins.length, 10); - - const reversed = await addrindexer.getCoinsByAddress( - addr, {reverse: true}); - - assert.strictEqual(reversed.length, 10); - - for (let i = 0; i < 10; i++) - assert.deepEqual(coins[i], reversed[9 - i]); - }); - it('should get txs by address (limit)', async () => { const addr = miner.getAddress(); const hashes = await addrindexer.getHashesByAddress(addr, {limit: 1}); @@ -156,47 +119,6 @@ describe('Indexer', function() { assert.deepEqual(hashes[i], reversed[9 - i]); }); - it('should coins by address after txid and index', async () => { - const addr = miner.getAddress(); - const coins = await addrindexer.getCoinsByAddress(addr, {limit: 5}); - - assert.strictEqual(coins.length, 5); - - const txid = coins[4].hash; - const index = coins[4].index; - - const next = await addrindexer.getCoinsByAddressAfter( - addr, {txid: txid, index: index, limit: 5}); - - assert.strictEqual(next.length, 5); - - const all = await addrindexer.getCoinsByAddress(addr); - assert.strictEqual(all.length, 10); - - assert.deepEqual(coins.concat(next), all); - }); - - it('should coins by address after txid and index (reverse)', async () => { - const addr = miner.getAddress(); - const coins = await addrindexer.getCoinsByAddress( - addr, {limit: 5, reverse: true}); - - assert.strictEqual(coins.length, 5); - - const txid = coins[4].hash; - const index = coins[4].index; - - const next = await addrindexer.getCoinsByAddressAfter( - addr, {txid: txid, index: index, limit: 5, reverse: true}); - - assert.strictEqual(next.length, 5); - - const all = await addrindexer.getCoinsByAddress(addr, {reverse: true}); - assert.strictEqual(all.length, 10); - - assert.deepEqual(coins.concat(next), all); - }); - it('should txs by address after txid', async () => { const addr = miner.getAddress(); const hashes = await addrindexer.getHashesByAddress(addr, {limit: 5}); @@ -251,12 +173,12 @@ describe('Indexer', function() { assert.strictEqual(txindexer.state.startHeight, 20); assert.strictEqual(addrindexer.state.startHeight, 20); - const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 20); + const hashes = await addrindexer.getHashesByAddress(miner.getAddress()); + assert.strictEqual(hashes.length, 20); - for (const coin of coins) { - const meta = await txindexer.getMeta(coin.hash); - assert.bufferEqual(meta.tx.hash(), coin.hash); + for (const hash of hashes) { + const meta = await txindexer.getMeta(hash); + assert.bufferEqual(meta.tx.hash(), hash); } }); @@ -266,12 +188,12 @@ describe('Indexer', function() { assert.strictEqual(txindexer.state.startHeight, 31); assert.strictEqual(addrindexer.state.startHeight, 31); - const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); - assert.strictEqual(coins.length, 31); + const hashes = await addrindexer.getHashesByAddress(miner.getAddress()); + assert.strictEqual(hashes.length, 31); - for (const coin of coins) { - const meta = await txindexer.getMeta(coin.hash); - assert.bufferEqual(meta.tx.hash(), coin.hash); + for (const hash of hashes) { + const meta = await txindexer.getMeta(hash); + assert.bufferEqual(meta.tx.hash(), hash); } }); }); diff --git a/test/node-test.js b/test/node-test.js index fa017fa1..9d2c37ef 100644 --- a/test/node-test.js +++ b/test/node-test.js @@ -775,38 +775,6 @@ describe('Node', function() { assert.strictEqual(meta.tx.txid(), tx2.txid()); }); - it('should get coin/tx by addr', async () => { - const addr = await wallet.receiveAddress(); - const mtx = await wallet.createTX({ - rate: 100000, - outputs: [{ - value: 100000, - address: addr - }] - }); - - await wallet.sign(mtx); - - const tx = mtx.toTX(); - const job = await miner.createJob(); - - job.addTX(tx, mtx.view); - job.refresh(); - - const block = await job.mineAsync(); - await chain.add(block); - - await new Promise(r => setTimeout(r, 300)); - - const txs = await node.getTXByAddress(addr.hash); - const tx2 = txs[0]; - assert.strictEqual(tx.txid(), tx2.txid()); - - const coins = await node.getCoinsByAddress(addr.hash); - const coin = coins[0]; - assert.strictEqual(tx.txid(), coin.txid()); - }); - it('should cleanup', async () => { consensus.COINBASE_MATURITY = 100; await node.close();