diff --git a/lib/indexer/addrindexer.js b/lib/indexer/addrindexer.js index b6112226..6f463c07 100644 --- a/lib/indexer/addrindexer.js +++ b/lib/indexer/addrindexer.js @@ -222,11 +222,14 @@ class AddrIndexer extends Indexer { } }; - if (after) { - const raw = await this.db.get(layout.c.encode(after)); - if (!raw) - return []; + const hasAfter = (after && await this.db.has(layout.c.encode(after))); + const skip = (after && !hasAfter && !reverse); + if (skip) + return []; + + if (after && hasAfter) { + const raw = await this.db.get(layout.c.encode(after)); const count = Count.fromRaw(raw); const {height, index} = count; diff --git a/lib/mempool/mempool.js b/lib/mempool/mempool.js index fb1bcd1e..7410a5d0 100644 --- a/lib/mempool/mempool.js +++ b/lib/mempool/mempool.js @@ -19,7 +19,6 @@ const policy = require('../protocol/policy'); const util = require('../utils/util'); const random = require('bcrypto/lib/random'); const {VerifyError} = require('../protocol/errors'); -const Address = require('../primitives/address'); const Script = require('../script/script'); const Outpoint = require('../primitives/outpoint'); const TX = require('../primitives/tx'); @@ -73,8 +72,7 @@ class Mempool extends EventEmitter { this.spents = new BufferMap(); this.rejects = new RollingFilter(120000, 0.000001); - this.coinIndex = new CoinIndex(); - this.txIndex = new TXIndex(); + this.addrindex = new AddrIndex(); } /** @@ -364,8 +362,7 @@ class Mempool extends EventEmitter { this.orphans.clear(); this.map.clear(); this.spents.clear(); - this.coinIndex.reset(); - this.txIndex.reset(); + this.addrindex.reset(); this.freeCount = 0; this.lastTime = 0; @@ -568,73 +565,32 @@ class Mempool extends EventEmitter { return entry.tx; } - /** - * Find all coins pertaining to a certain address. - * @param {Address[]} addrs - * @returns {Coin[]} - */ - - getCoinsByAddress(addrs) { - if (!Array.isArray(addrs)) - addrs = [addrs]; - - const out = []; - - for (const addr of addrs) { - const hash = Address.getHash(addr); - const coins = this.coinIndex.get(hash); - - for (const coin of coins) - out.push(coin); - } - - return out; - } - /** * Find all transactions pertaining to a certain address. - * @param {Address[]} addrs + * @param {Address} addr + * @param {Object} options + * @param {Number} options.limit + * @param {Number} options.reverse + * @param {Buffer} options.after * @returns {TX[]} */ - getTXByAddress(addrs) { - if (!Array.isArray(addrs)) - addrs = [addrs]; - - const out = []; - - for (const addr of addrs) { - const hash = Address.getHash(addr); - const txs = this.txIndex.get(hash); - - for (const tx of txs) - out.push(tx); - } - - return out; + getTXByAddress(addr, options) { + return this.addrindex.get(addr, options); } /** * Find all transactions pertaining to a certain address. - * @param {Address[]} addrs + * @param {Address} addr + * @param {Object} options + * @param {Number} options.limit + * @param {Number} options.reverse + * @param {Buffer} options.after * @returns {TXMeta[]} */ - getMetaByAddress(addrs) { - if (!Array.isArray(addrs)) - addrs = [addrs]; - - const out = []; - - for (const addr of addrs) { - const hash = Address.getHash(addr); - const txs = this.txIndex.getMeta(hash); - - for (const tx of txs) - out.push(tx); - } - - return out; + getMetaByAddress(addr, options) { + return this.addrindex.getMeta(addr, options); } /** @@ -1883,17 +1839,7 @@ class Mempool extends EventEmitter { */ indexEntry(entry, view) { - const tx = entry.tx; - - this.txIndex.insert(entry, view); - - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - this.coinIndex.remove(hash, index); - } - - for (let i = 0; i < tx.outputs.length; i++) - this.coinIndex.insert(tx, i); + this.addrindex.insert(entry, view); } /** @@ -1903,23 +1849,8 @@ class Mempool extends EventEmitter { */ unindexEntry(entry) { - const tx = entry.tx; - const hash = tx.hash(); - - this.txIndex.remove(hash); - - for (const {prevout} of tx.inputs) { - const {hash, index} = prevout; - const prev = this.getTX(hash); - - if (!prev) - continue; - - this.coinIndex.insert(prev, index); - } - - for (let i = 0; i < tx.outputs.length; i++) - this.coinIndex.remove(hash, i); + const hash = entry.tx.hash(); + this.addrindex.remove(hash); } /** @@ -2188,11 +2119,11 @@ class MempoolOptions { } /** - * TX Address Index + * Address Index * @ignore */ -class TXIndex { +class AddrIndex { /** * Create TX address index. * @constructor @@ -2211,29 +2142,53 @@ class TXIndex { this.map.clear(); } - get(addr) { - const items = this.index.get(addr); + getKey(addr) { + const prefix = addr.getPrefix(); - if (!items) - return []; + if (prefix < 0) + return null; + + const raw = Buffer.allocUnsafe(1); + raw.writeUInt8(prefix); + + return Buffer.concat([raw, addr.getHash()]); + } + + /** + * Get transactions by address. + * @param {Address} addr + * @param {Object} options + * @param {Number} options.limit + * @param {Number} options.reverse + * @param {Buffer} options.after + */ + + get(addr, options = {}) { + const values = this.getEntries(addr, options); const out = []; - for (const entry of items.values()) + for (const entry of values) out.push(entry.tx); return out; } - getMeta(addr) { - const items = this.index.get(addr); + /** + * Get transaction meta by address. + * @param {Address} addr + * @param {Object} options + * @param {Number} options.limit + * @param {Number} options.reverse + * @param {Buffer} options.after + */ - if (!items) - return []; + getMeta(addr, options = {}) { + const values = this.getEntries(addr, options); const out = []; - for (const entry of items.values()) { + for (const entry of values) { const meta = TXMeta.fromTX(entry.tx); meta.mtime = entry.time; out.push(meta); @@ -2242,20 +2197,101 @@ class TXIndex { return out; } + /** + * Get entries by address. + * @param {Address} addr + * @param {Object} options + * @param {Number} options.limit + * @param {Number} options.reverse + * @param {Buffer} options.after + */ + + getEntries(addr, options = {}) { + const {limit, reverse, after} = options; + const key = this.getKey(addr); + + if (!key) + return []; + + const items = this.index.get(key); + + if (!items) + return []; + + let values = []; + + const skip = (after && !items.has(after) && reverse); + + if (skip) + return values; + + if (after && items.has(after)) { + let index = 0; + + for (const k of items.keys()) { + if (k.compare(after) === 0) + break; + index += 1; + } + + values = Array.from(items.values()); + + let start = index + 1; + let end = values.length; + + if (end - start > limit) + end = start + limit; + + if (reverse) { + start = 0; + end = index; + + if (end - start > limit) + start = end - limit; + } + + values = values.slice(start, end); + } else { + values = Array.from(items.values()); + + if (values.length > limit) { + let start = 0; + let end = limit; + + if (reverse) { + start = values.length - limit; + end = values.length; + } + + values = values.slice(start, end); + } + } + + if (reverse) + values.reverse(); + + return values; + } + insert(entry, view) { const tx = entry.tx; const hash = tx.hash(); - const addrs = tx.getHashes(view); + const addrs = tx.getAddresses(view); if (addrs.length === 0) return; for (const addr of addrs) { - let items = this.index.get(addr); + const key = this.getKey(addr); + + if (!key) + continue; + + let items = this.index.get(key); if (!items) { items = new BufferMap(); - this.index.set(addr, items); + this.index.set(key, items); } assert(!items.has(hash)); @@ -2272,7 +2308,12 @@ class TXIndex { return; for (const addr of addrs) { - const items = this.index.get(addr); + const key = this.getKey(addr); + + if (!key) + continue; + + const items = this.index.get(key); assert(items); assert(items.has(hash)); @@ -2280,117 +2321,13 @@ class TXIndex { items.delete(hash); if (items.size === 0) - this.index.delete(addr); + this.index.delete(key); } this.map.delete(hash); } } -/** - * Coin Address Index - * @ignore - */ - -class CoinIndex { - /** - * Create coin address index. - * @constructor - */ - - constructor() { - // Map of addr->coins. - this.index = new BufferMap(); - - // Map of outpoint->addr. - this.map = new BufferMap(); - } - - reset() { - this.index.clear(); - this.map.clear(); - } - - get(addr) { - const items = this.index.get(addr); - - if (!items) - return []; - - const out = []; - - for (const coin of items.values()) - out.push(coin.toCoin()); - - return out; - } - - insert(tx, index) { - const output = tx.outputs[index]; - const hash = tx.hash(); - const addr = output.getHash(); - - if (!addr) - return; - - let items = this.index.get(addr); - - if (!items) { - items = new BufferMap(); - this.index.set(addr, items); - } - - const key = Outpoint.toKey(hash, index); - - assert(!items.has(key)); - items.set(key, new IndexedCoin(tx, index)); - - this.map.set(key, addr); - } - - remove(hash, index) { - const key = Outpoint.toKey(hash, index); - const addr = this.map.get(key); - - if (!addr) - return; - - const items = this.index.get(addr); - - assert(items); - assert(items.has(key)); - items.delete(key); - - if (items.size === 0) - this.index.delete(addr); - - this.map.delete(key); - } -} - -/** - * Indexed Coin - * @ignore - */ - -class IndexedCoin { - /** - * Create an indexed coin. - * @constructor - * @param {TX} tx - * @param {Number} index - */ - - constructor(tx, index) { - this.tx = tx; - this.index = index; - } - - toCoin() { - return Coin.fromTX(this.tx, this.index, -1); - } -} - /** * Orphan * @ignore diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 72b96b96..0a3612fc 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -471,21 +471,49 @@ class FullNode extends Node { */ async getMetaByAddress(addr, options = {}) { - const mempool = this.mempool.getMetaByAddress(addr); + if (!this.txindex || !this.addrindex) + return []; - if (this.txindex && this.addrindex) { - const hashes = await this.addrindex.getHashesByAddress(addr, options); - const mtxs = []; + const {reverse, after} = options; + let {limit} = options; + + let metas = []; + + const confirmed = async () => { + const hashes = await this.addrindex.getHashesByAddress( + addr, {limit, reverse, after}); for (const hash of hashes) { const mtx = await this.txindex.getMeta(hash); assert(mtx); - mtxs.push(mtx); + metas.push(mtx); } - return mtxs.concat(mempool); - } + }; - return mempool; + const unconfirmed = () => { + const mempool = this.mempool.getMetaByAddress( + addr, {limit, reverse, after}); + + metas = metas.concat(mempool); + }; + + if (reverse) + unconfirmed(); + else + await confirmed(); + + if (metas.length > 0) + limit -= metas.length; + + if (limit <= 0) + return metas; + + if (reverse) + await confirmed(); + else + unconfirmed(); + + return metas; } /** diff --git a/test/indexer-test.js b/test/indexer-test.js index cd45499c..56285944 100644 --- a/test/indexer-test.js +++ b/test/indexer-test.js @@ -220,7 +220,8 @@ describe('Indexer', function() { } ]; - const txids = []; + const confirmed = []; + const unconfirmed = []; const ports = { p2p: 49331, @@ -287,7 +288,7 @@ describe('Indexer', function() { const txid = await wclient.execute( 'sendtoaddress', [v.addr, v.amount]); - txids.push(txid); + confirmed.push(txid); } const blocks = await nclient.execute( @@ -295,6 +296,16 @@ describe('Indexer', function() { assert.equal(blocks.length, 1); } + + // Send unconfirmed to the vector addresses. + for (let i = 0; i < 3; i++) { + for (const v of vectors) { + const txid = await wclient.execute( + 'sendtoaddress', [v.addr, v.amount]); + + unconfirmed.push(txid); + } + } }); after(async () => { @@ -304,36 +315,69 @@ describe('Indexer', function() { }); for (const v of vectors) { - it(`will get txs by ${v.label} address`, async () => { + it(`txs by ${v.label} address`, async () => { const res = await nclient.request( 'GET', `/tx/address/${v.addr}`, {}); - assert.equal(res.length, 10); + assert.equal(res.length, 13); - for (const tx of res) - assert(txids.includes(tx.hash)); + for (let i = 0; i < 10; i++) + assert(confirmed.includes(res[i].hash)); + + for (let i = 10; i < 13; i++) + assert(unconfirmed.includes(res[i].hash)); }); - it(`will get txs by ${v.label} address (limit)`, async () => { + it(`txs by ${v.label} address (limit)`, async () => { const res = await nclient.request( 'GET', `/tx/address/${v.addr}`, {limit: 3}); + assert.equal(res.length, 3); + for (const tx of res) - assert(txids.includes(tx.hash)); + assert(confirmed.includes(tx.hash)); + }); + + it(`txs by ${v.label} address (limit w/ unconf)`, async () => { + const res = await nclient.request( + 'GET', `/tx/address/${v.addr}`, {limit: 11}); + + assert.equal(res.length, 11); + + for (let i = 0; i < 10; i++) + assert(confirmed.includes(res[i].hash)); + + for (let i = 10; i < 11; i++) + assert(unconfirmed.includes(res[i].hash)); }); it(`txs by ${v.label} address (reverse)`, async () => { const asc = await nclient.request( 'GET', `/tx/address/${v.addr}`, {reverse: false}); + assert.equal(asc.length, 13); + const dsc = await nclient.request( 'GET', `/tx/address/${v.addr}`, {reverse: true}); - for (let i = 0; i < dsc.length; i++) - assert.equal(asc[i].hash, dsc[dsc.length - i - 1].hash); + assert.equal(asc.length, 13); + + for (let i = 0; i < 10; i++) + assert(confirmed.includes(asc[i].hash)); + + for (let i = 10; i < 13; i++) + assert(unconfirmed.includes(asc[i].hash)); + + // Check the the results are reverse + // of each other. + for (let i = 0; i < dsc.length; i++) { + const atx = asc[i]; + const dtx = dsc[dsc.length - i - 1]; + assert.equal(atx.hash, dtx.hash); + } }); - it(`txs by ${v.label} address after txid`, async () => { + it(`txs by ${v.label} address (after)`, async () => { const one = await nclient.request( 'GET', `/tx/address/${v.addr}`, {limit: 3}); assert.strictEqual(one.length, 3); @@ -351,26 +395,105 @@ describe('Indexer', function() { assert.deepEqual(one.concat(two), all); }); - it(`txs by ${v.label} address after txid (reverse)`, async () => { + it(`txs by ${v.label} address (after w/ unconf)`, async () => { + const one = await nclient.request( + 'GET', `/tx/address/${v.addr}`, {limit: 11}); + assert.strictEqual(one.length, 11); + + for (let i = 0; i < 10; i++) + assert(confirmed.includes(one[i].hash)); + + for (let i = 10; i < 11; i++) + assert(unconfirmed.includes(one[i].hash)); + + // The after hash is within the + // unconfirmed transactions. + const hash = one[10].hash; + + const two = await nclient.request( + 'GET', `/tx/address/${v.addr}`, {after: hash, limit: 1}); + assert.strictEqual(two.length, 1); + assert(unconfirmed.includes(two[0].hash)); + + const all = await nclient.request( + 'GET', `/tx/address/${v.addr}`, {limit: 12}); + assert.strictEqual(all.length, 12); + + assert.deepEqual(one.concat(two), all); + }); + + it(`txs by ${v.label} address (after, reverse)`, async () => { + const one = await nclient.request( + 'GET', `/tx/address/${v.addr}`, + {limit: 5, reverse: true}); + + assert.strictEqual(one.length, 5); + + for (let i = 0; i < 3; i++) + assert(unconfirmed.includes(one[i].hash)); + + for (let i = 3; i < 5; i++) + assert(confirmed.includes(one[i].hash)); + + // The after hash is within the + // confirmed transactions. + const hash = one[4].hash; + + const two = await nclient.request( + 'GET', `/tx/address/${v.addr}`, + {after: hash, limit: 3, reverse: true}); + + assert.strictEqual(two.length, 3); + for (let i = 0; i < 3; i++) + assert(confirmed.includes(two[i].hash)); + + const all = await nclient.request( + 'GET', `/tx/address/${v.addr}`, + {limit: 8, reverse: true}); + + assert.strictEqual(all.length, 8); + + for (let i = 0; i < 3; i++) + assert(unconfirmed.includes(all[i].hash)); + + for (let i = 3; i < 8; i++) + assert(confirmed.includes(all[i].hash)); + + assert.deepEqual(one.concat(two), all); + }); + + it(`txs by ${v.label} address (after, reverse w/ unconf)`, async () => { const one = await nclient.request( 'GET', `/tx/address/${v.addr}`, {limit: 3, reverse: true}); assert.strictEqual(one.length, 3); + for (let i = 0; i < 3; i++) + assert(unconfirmed.includes(one[i].hash)); + // The after hash is within the + // unconfirmed transactions. const hash = one[2].hash; const two = await nclient.request( 'GET', `/tx/address/${v.addr}`, {after: hash, limit: 3, reverse: true}); - assert.strictEqual(one.length, 3); + assert.strictEqual(two.length, 3); + for (let i = 0; i < 3; i++) + assert(confirmed.includes(two[i].hash)); const all = await nclient.request( 'GET', `/tx/address/${v.addr}`, {limit: 6, reverse: true}); - assert.strictEqual(one.length, 3); + assert.strictEqual(all.length, 6); + + for (let i = 0; i < 3; i++) + assert(unconfirmed.includes(all[i].hash)); + + for (let i = 3; i < 6; i++) + assert(confirmed.includes(all[i].hash)); assert.deepEqual(one.concat(two), all); });