mempool: update for addrindex

This commit is contained in:
Braydon Fuller 2019-04-16 11:11:36 -07:00
parent 4c8f11ed34
commit bd26dbf32d
No known key found for this signature in database
GPG Key ID: F24F232D108B3AD4
4 changed files with 326 additions and 235 deletions

View File

@ -222,11 +222,14 @@ class AddrIndexer extends Indexer {
}
};
if (after) {
const raw = await this.db.get(layout.c.encode(after));
if (!raw)
return [];
const hasAfter = (after && await this.db.has(layout.c.encode(after)));
const skip = (after && !hasAfter && !reverse);
if (skip)
return [];
if (after && hasAfter) {
const raw = await this.db.get(layout.c.encode(after));
const count = Count.fromRaw(raw);
const {height, index} = count;

View File

@ -19,7 +19,6 @@ const policy = require('../protocol/policy');
const util = require('../utils/util');
const random = require('bcrypto/lib/random');
const {VerifyError} = require('../protocol/errors');
const Address = require('../primitives/address');
const Script = require('../script/script');
const Outpoint = require('../primitives/outpoint');
const TX = require('../primitives/tx');
@ -73,8 +72,7 @@ class Mempool extends EventEmitter {
this.spents = new BufferMap();
this.rejects = new RollingFilter(120000, 0.000001);
this.coinIndex = new CoinIndex();
this.txIndex = new TXIndex();
this.addrindex = new AddrIndex();
}
/**
@ -364,8 +362,7 @@ class Mempool extends EventEmitter {
this.orphans.clear();
this.map.clear();
this.spents.clear();
this.coinIndex.reset();
this.txIndex.reset();
this.addrindex.reset();
this.freeCount = 0;
this.lastTime = 0;
@ -568,73 +565,32 @@ class Mempool extends EventEmitter {
return entry.tx;
}
/**
* Find all coins pertaining to a certain address.
* @param {Address[]} addrs
* @returns {Coin[]}
*/
getCoinsByAddress(addrs) {
if (!Array.isArray(addrs))
addrs = [addrs];
const out = [];
for (const addr of addrs) {
const hash = Address.getHash(addr);
const coins = this.coinIndex.get(hash);
for (const coin of coins)
out.push(coin);
}
return out;
}
/**
* Find all transactions pertaining to a certain address.
* @param {Address[]} addrs
* @param {Address} addr
* @param {Object} options
* @param {Number} options.limit
* @param {Number} options.reverse
* @param {Buffer} options.after
* @returns {TX[]}
*/
getTXByAddress(addrs) {
if (!Array.isArray(addrs))
addrs = [addrs];
const out = [];
for (const addr of addrs) {
const hash = Address.getHash(addr);
const txs = this.txIndex.get(hash);
for (const tx of txs)
out.push(tx);
}
return out;
getTXByAddress(addr, options) {
return this.addrindex.get(addr, options);
}
/**
* Find all transactions pertaining to a certain address.
* @param {Address[]} addrs
* @param {Address} addr
* @param {Object} options
* @param {Number} options.limit
* @param {Number} options.reverse
* @param {Buffer} options.after
* @returns {TXMeta[]}
*/
getMetaByAddress(addrs) {
if (!Array.isArray(addrs))
addrs = [addrs];
const out = [];
for (const addr of addrs) {
const hash = Address.getHash(addr);
const txs = this.txIndex.getMeta(hash);
for (const tx of txs)
out.push(tx);
}
return out;
getMetaByAddress(addr, options) {
return this.addrindex.getMeta(addr, options);
}
/**
@ -1883,17 +1839,7 @@ class Mempool extends EventEmitter {
*/
indexEntry(entry, view) {
const tx = entry.tx;
this.txIndex.insert(entry, view);
for (const {prevout} of tx.inputs) {
const {hash, index} = prevout;
this.coinIndex.remove(hash, index);
}
for (let i = 0; i < tx.outputs.length; i++)
this.coinIndex.insert(tx, i);
this.addrindex.insert(entry, view);
}
/**
@ -1903,23 +1849,8 @@ class Mempool extends EventEmitter {
*/
unindexEntry(entry) {
const tx = entry.tx;
const hash = tx.hash();
this.txIndex.remove(hash);
for (const {prevout} of tx.inputs) {
const {hash, index} = prevout;
const prev = this.getTX(hash);
if (!prev)
continue;
this.coinIndex.insert(prev, index);
}
for (let i = 0; i < tx.outputs.length; i++)
this.coinIndex.remove(hash, i);
const hash = entry.tx.hash();
this.addrindex.remove(hash);
}
/**
@ -2188,11 +2119,11 @@ class MempoolOptions {
}
/**
* TX Address Index
* Address Index
* @ignore
*/
class TXIndex {
class AddrIndex {
/**
* Create TX address index.
* @constructor
@ -2211,29 +2142,53 @@ class TXIndex {
this.map.clear();
}
get(addr) {
const items = this.index.get(addr);
getKey(addr) {
const prefix = addr.getPrefix();
if (!items)
return [];
if (prefix < 0)
return null;
const raw = Buffer.allocUnsafe(1);
raw.writeUInt8(prefix);
return Buffer.concat([raw, addr.getHash()]);
}
/**
* Get transactions by address.
* @param {Address} addr
* @param {Object} options
* @param {Number} options.limit
* @param {Number} options.reverse
* @param {Buffer} options.after
*/
get(addr, options = {}) {
const values = this.getEntries(addr, options);
const out = [];
for (const entry of items.values())
for (const entry of values)
out.push(entry.tx);
return out;
}
getMeta(addr) {
const items = this.index.get(addr);
/**
* Get transaction meta by address.
* @param {Address} addr
* @param {Object} options
* @param {Number} options.limit
* @param {Number} options.reverse
* @param {Buffer} options.after
*/
if (!items)
return [];
getMeta(addr, options = {}) {
const values = this.getEntries(addr, options);
const out = [];
for (const entry of items.values()) {
for (const entry of values) {
const meta = TXMeta.fromTX(entry.tx);
meta.mtime = entry.time;
out.push(meta);
@ -2242,20 +2197,101 @@ class TXIndex {
return out;
}
/**
* Get entries by address.
* @param {Address} addr
* @param {Object} options
* @param {Number} options.limit
* @param {Number} options.reverse
* @param {Buffer} options.after
*/
getEntries(addr, options = {}) {
const {limit, reverse, after} = options;
const key = this.getKey(addr);
if (!key)
return [];
const items = this.index.get(key);
if (!items)
return [];
let values = [];
const skip = (after && !items.has(after) && reverse);
if (skip)
return values;
if (after && items.has(after)) {
let index = 0;
for (const k of items.keys()) {
if (k.compare(after) === 0)
break;
index += 1;
}
values = Array.from(items.values());
let start = index + 1;
let end = values.length;
if (end - start > limit)
end = start + limit;
if (reverse) {
start = 0;
end = index;
if (end - start > limit)
start = end - limit;
}
values = values.slice(start, end);
} else {
values = Array.from(items.values());
if (values.length > limit) {
let start = 0;
let end = limit;
if (reverse) {
start = values.length - limit;
end = values.length;
}
values = values.slice(start, end);
}
}
if (reverse)
values.reverse();
return values;
}
insert(entry, view) {
const tx = entry.tx;
const hash = tx.hash();
const addrs = tx.getHashes(view);
const addrs = tx.getAddresses(view);
if (addrs.length === 0)
return;
for (const addr of addrs) {
let items = this.index.get(addr);
const key = this.getKey(addr);
if (!key)
continue;
let items = this.index.get(key);
if (!items) {
items = new BufferMap();
this.index.set(addr, items);
this.index.set(key, items);
}
assert(!items.has(hash));
@ -2272,7 +2308,12 @@ class TXIndex {
return;
for (const addr of addrs) {
const items = this.index.get(addr);
const key = this.getKey(addr);
if (!key)
continue;
const items = this.index.get(key);
assert(items);
assert(items.has(hash));
@ -2280,117 +2321,13 @@ class TXIndex {
items.delete(hash);
if (items.size === 0)
this.index.delete(addr);
this.index.delete(key);
}
this.map.delete(hash);
}
}
/**
* Coin Address Index
* @ignore
*/
class CoinIndex {
/**
* Create coin address index.
* @constructor
*/
constructor() {
// Map of addr->coins.
this.index = new BufferMap();
// Map of outpoint->addr.
this.map = new BufferMap();
}
reset() {
this.index.clear();
this.map.clear();
}
get(addr) {
const items = this.index.get(addr);
if (!items)
return [];
const out = [];
for (const coin of items.values())
out.push(coin.toCoin());
return out;
}
insert(tx, index) {
const output = tx.outputs[index];
const hash = tx.hash();
const addr = output.getHash();
if (!addr)
return;
let items = this.index.get(addr);
if (!items) {
items = new BufferMap();
this.index.set(addr, items);
}
const key = Outpoint.toKey(hash, index);
assert(!items.has(key));
items.set(key, new IndexedCoin(tx, index));
this.map.set(key, addr);
}
remove(hash, index) {
const key = Outpoint.toKey(hash, index);
const addr = this.map.get(key);
if (!addr)
return;
const items = this.index.get(addr);
assert(items);
assert(items.has(key));
items.delete(key);
if (items.size === 0)
this.index.delete(addr);
this.map.delete(key);
}
}
/**
* Indexed Coin
* @ignore
*/
class IndexedCoin {
/**
* Create an indexed coin.
* @constructor
* @param {TX} tx
* @param {Number} index
*/
constructor(tx, index) {
this.tx = tx;
this.index = index;
}
toCoin() {
return Coin.fromTX(this.tx, this.index, -1);
}
}
/**
* Orphan
* @ignore

View File

@ -471,21 +471,49 @@ class FullNode extends Node {
*/
async getMetaByAddress(addr, options = {}) {
const mempool = this.mempool.getMetaByAddress(addr);
if (!this.txindex || !this.addrindex)
return [];
if (this.txindex && this.addrindex) {
const hashes = await this.addrindex.getHashesByAddress(addr, options);
const mtxs = [];
const {reverse, after} = options;
let {limit} = options;
let metas = [];
const confirmed = async () => {
const hashes = await this.addrindex.getHashesByAddress(
addr, {limit, reverse, after});
for (const hash of hashes) {
const mtx = await this.txindex.getMeta(hash);
assert(mtx);
mtxs.push(mtx);
metas.push(mtx);
}
return mtxs.concat(mempool);
}
};
return mempool;
const unconfirmed = () => {
const mempool = this.mempool.getMetaByAddress(
addr, {limit, reverse, after});
metas = metas.concat(mempool);
};
if (reverse)
unconfirmed();
else
await confirmed();
if (metas.length > 0)
limit -= metas.length;
if (limit <= 0)
return metas;
if (reverse)
await confirmed();
else
unconfirmed();
return metas;
}
/**

View File

@ -220,7 +220,8 @@ describe('Indexer', function() {
}
];
const txids = [];
const confirmed = [];
const unconfirmed = [];
const ports = {
p2p: 49331,
@ -287,7 +288,7 @@ describe('Indexer', function() {
const txid = await wclient.execute(
'sendtoaddress', [v.addr, v.amount]);
txids.push(txid);
confirmed.push(txid);
}
const blocks = await nclient.execute(
@ -295,6 +296,16 @@ describe('Indexer', function() {
assert.equal(blocks.length, 1);
}
// Send unconfirmed to the vector addresses.
for (let i = 0; i < 3; i++) {
for (const v of vectors) {
const txid = await wclient.execute(
'sendtoaddress', [v.addr, v.amount]);
unconfirmed.push(txid);
}
}
});
after(async () => {
@ -304,36 +315,69 @@ describe('Indexer', function() {
});
for (const v of vectors) {
it(`will get txs by ${v.label} address`, async () => {
it(`txs by ${v.label} address`, async () => {
const res = await nclient.request(
'GET', `/tx/address/${v.addr}`, {});
assert.equal(res.length, 10);
assert.equal(res.length, 13);
for (const tx of res)
assert(txids.includes(tx.hash));
for (let i = 0; i < 10; i++)
assert(confirmed.includes(res[i].hash));
for (let i = 10; i < 13; i++)
assert(unconfirmed.includes(res[i].hash));
});
it(`will get txs by ${v.label} address (limit)`, async () => {
it(`txs by ${v.label} address (limit)`, async () => {
const res = await nclient.request(
'GET', `/tx/address/${v.addr}`, {limit: 3});
assert.equal(res.length, 3);
for (const tx of res)
assert(txids.includes(tx.hash));
assert(confirmed.includes(tx.hash));
});
it(`txs by ${v.label} address (limit w/ unconf)`, async () => {
const res = await nclient.request(
'GET', `/tx/address/${v.addr}`, {limit: 11});
assert.equal(res.length, 11);
for (let i = 0; i < 10; i++)
assert(confirmed.includes(res[i].hash));
for (let i = 10; i < 11; i++)
assert(unconfirmed.includes(res[i].hash));
});
it(`txs by ${v.label} address (reverse)`, async () => {
const asc = await nclient.request(
'GET', `/tx/address/${v.addr}`, {reverse: false});
assert.equal(asc.length, 13);
const dsc = await nclient.request(
'GET', `/tx/address/${v.addr}`, {reverse: true});
for (let i = 0; i < dsc.length; i++)
assert.equal(asc[i].hash, dsc[dsc.length - i - 1].hash);
assert.equal(asc.length, 13);
for (let i = 0; i < 10; i++)
assert(confirmed.includes(asc[i].hash));
for (let i = 10; i < 13; i++)
assert(unconfirmed.includes(asc[i].hash));
// Check the the results are reverse
// of each other.
for (let i = 0; i < dsc.length; i++) {
const atx = asc[i];
const dtx = dsc[dsc.length - i - 1];
assert.equal(atx.hash, dtx.hash);
}
});
it(`txs by ${v.label} address after txid`, async () => {
it(`txs by ${v.label} address (after)`, async () => {
const one = await nclient.request(
'GET', `/tx/address/${v.addr}`, {limit: 3});
assert.strictEqual(one.length, 3);
@ -351,26 +395,105 @@ describe('Indexer', function() {
assert.deepEqual(one.concat(two), all);
});
it(`txs by ${v.label} address after txid (reverse)`, async () => {
it(`txs by ${v.label} address (after w/ unconf)`, async () => {
const one = await nclient.request(
'GET', `/tx/address/${v.addr}`, {limit: 11});
assert.strictEqual(one.length, 11);
for (let i = 0; i < 10; i++)
assert(confirmed.includes(one[i].hash));
for (let i = 10; i < 11; i++)
assert(unconfirmed.includes(one[i].hash));
// The after hash is within the
// unconfirmed transactions.
const hash = one[10].hash;
const two = await nclient.request(
'GET', `/tx/address/${v.addr}`, {after: hash, limit: 1});
assert.strictEqual(two.length, 1);
assert(unconfirmed.includes(two[0].hash));
const all = await nclient.request(
'GET', `/tx/address/${v.addr}`, {limit: 12});
assert.strictEqual(all.length, 12);
assert.deepEqual(one.concat(two), all);
});
it(`txs by ${v.label} address (after, reverse)`, async () => {
const one = await nclient.request(
'GET', `/tx/address/${v.addr}`,
{limit: 5, reverse: true});
assert.strictEqual(one.length, 5);
for (let i = 0; i < 3; i++)
assert(unconfirmed.includes(one[i].hash));
for (let i = 3; i < 5; i++)
assert(confirmed.includes(one[i].hash));
// The after hash is within the
// confirmed transactions.
const hash = one[4].hash;
const two = await nclient.request(
'GET', `/tx/address/${v.addr}`,
{after: hash, limit: 3, reverse: true});
assert.strictEqual(two.length, 3);
for (let i = 0; i < 3; i++)
assert(confirmed.includes(two[i].hash));
const all = await nclient.request(
'GET', `/tx/address/${v.addr}`,
{limit: 8, reverse: true});
assert.strictEqual(all.length, 8);
for (let i = 0; i < 3; i++)
assert(unconfirmed.includes(all[i].hash));
for (let i = 3; i < 8; i++)
assert(confirmed.includes(all[i].hash));
assert.deepEqual(one.concat(two), all);
});
it(`txs by ${v.label} address (after, reverse w/ unconf)`, async () => {
const one = await nclient.request(
'GET', `/tx/address/${v.addr}`,
{limit: 3, reverse: true});
assert.strictEqual(one.length, 3);
for (let i = 0; i < 3; i++)
assert(unconfirmed.includes(one[i].hash));
// The after hash is within the
// unconfirmed transactions.
const hash = one[2].hash;
const two = await nclient.request(
'GET', `/tx/address/${v.addr}`,
{after: hash, limit: 3, reverse: true});
assert.strictEqual(one.length, 3);
assert.strictEqual(two.length, 3);
for (let i = 0; i < 3; i++)
assert(confirmed.includes(two[i].hash));
const all = await nclient.request(
'GET', `/tx/address/${v.addr}`,
{limit: 6, reverse: true});
assert.strictEqual(one.length, 3);
assert.strictEqual(all.length, 6);
for (let i = 0; i < 3; i++)
assert(unconfirmed.includes(all[i].hash));
for (let i = 3; i < 6; i++)
assert(confirmed.includes(all[i].hash));
assert.deepEqual(one.concat(two), all);
});