indexer: remove address coin index
This commit is contained in:
Braydon Fuller
parent
05d55efb22
commit
7dc55c9c3c
@ -16,17 +16,24 @@ const Indexer = require('./indexer');
|
||||
|
||||
/*
|
||||
* AddrIndexer Database Layout:
|
||||
* T[addr-hash][height][tx-index][hash] -> dummy (tx by address)
|
||||
* C[addr-hash][height][tx-index][hash][coin-index] -> dummy (coin by address)
|
||||
* x[addr-hash][hash] -> height and tx-index for tx
|
||||
* y[addr-hash][hash][index] -> height, tx-index and coin-index for coin
|
||||
*/
|
||||
* A[addr-hash][height][index][hash] -> dummy (tx by address, height and index)
|
||||
* a[addr-hash][hash] -> (tx height and index by address and tx hash)
|
||||
*
|
||||
* The database layout is organized so that transactions are sorted in
|
||||
* the same order as the blocks (e.g. chronological order) using the block
|
||||
* height and transaction index. This provides the ability to query for
|
||||
* sets of transactions within that order. For a wallet that would like to
|
||||
* synchronize or rescan, this could be a query for all of the latest
|
||||
* transactions, but not for earlier transactions that are already known.
|
||||
* Furthermore, to be able to query for all transactions in multiple sets
|
||||
* without reference to height and index, there is a mapping from address
|
||||
* and tx hash to the height and index as an entry point to the
|
||||
* ordered transactions.
|
||||
*/
|
||||
|
||||
Object.assign(layout, {
|
||||
T: bdb.key('T', ['hash', 'uint32', 'uint32', 'hash256']),
|
||||
C: bdb.key('C', ['hash', 'uint32', 'uint32', 'hash256', 'uint32']),
|
||||
x: bdb.key('x', ['hash', 'hash256']),
|
||||
y: bdb.key('y', ['hash', 'hash256', 'uint32'])
|
||||
A: bdb.key('A', ['hash', 'uint32', 'uint32', 'hash256']),
|
||||
a: bdb.key('a', ['hash', 'hash256'])
|
||||
});
|
||||
|
||||
/**
|
||||
@ -42,15 +49,11 @@ class Count {
|
||||
*/
|
||||
|
||||
constructor(height, index, coin) {
|
||||
this.height = height >= 0 ? height : 0;
|
||||
this.index = index >= 0 ? index : 0;
|
||||
this.coin = coin >= 0 ? coin : -1;
|
||||
this.height = height || 0;
|
||||
this.index = index || 0;
|
||||
|
||||
assert((this.height >>> 0) === this.height);
|
||||
assert((this.index >>> 0) === this.index);
|
||||
|
||||
if (coin)
|
||||
assert((this.coin >>> 0) === this.coin);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -59,18 +62,11 @@ class Count {
|
||||
*/
|
||||
|
||||
toRaw() {
|
||||
let len = 8;
|
||||
if (this.coin >= 0)
|
||||
len += 4;
|
||||
|
||||
const bw = bio.write(len);
|
||||
const bw = bio.write(8);
|
||||
|
||||
bw.writeU32(this.height);
|
||||
bw.writeU32(this.index);
|
||||
|
||||
if (this.coin >= 0)
|
||||
bw.writeU32(this.coin);
|
||||
|
||||
return bw.render();
|
||||
}
|
||||
|
||||
@ -86,9 +82,6 @@ class Count {
|
||||
this.height = br.readU32();
|
||||
this.index = br.readU32();
|
||||
|
||||
if (br.left() >= 4)
|
||||
this.coin = br.readU32();
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
@ -121,7 +114,6 @@ class AddrIndexer extends Indexer {
|
||||
|
||||
this.db = bdb.create(this.options);
|
||||
this.maxTxs = options.maxTxs || 100;
|
||||
this.maxCoins = options.maxCoins || 500;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -143,37 +135,8 @@ class AddrIndexer extends Indexer {
|
||||
for (const addr of tx.getHashes(view)) {
|
||||
const count = new Count(height, i);
|
||||
|
||||
b.put(layout.T.encode(addr, height, i, hash), null);
|
||||
b.put(layout.x.encode(addr, hash), count.toRaw());
|
||||
}
|
||||
|
||||
if (!tx.isCoinbase()) {
|
||||
for (const {prevout} of tx.inputs) {
|
||||
const {hash, index} = prevout;
|
||||
const coin = view.getOutput(prevout);
|
||||
assert(coin);
|
||||
|
||||
const addr = coin.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
b.del(layout.C.encode(addr, height, i, hash, index));
|
||||
b.del(layout.y.encode(addr, hash, index));
|
||||
}
|
||||
}
|
||||
|
||||
for (let j = 0; j < tx.outputs.length; j++) {
|
||||
const output = tx.outputs[j];
|
||||
const addr = output.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
const count = new Count(height, i, j);
|
||||
|
||||
b.put(layout.C.encode(addr, height, i, hash, j), null);
|
||||
b.put(layout.y.encode(addr, hash, j), count.toRaw());
|
||||
b.put(layout.A.encode(addr, height, i, hash), null);
|
||||
b.put(layout.a.encode(addr, hash), count.toRaw());
|
||||
}
|
||||
}
|
||||
|
||||
@ -197,147 +160,14 @@ class AddrIndexer extends Indexer {
|
||||
const hash = tx.hash();
|
||||
|
||||
for (const addr of tx.getHashes(view)) {
|
||||
b.del(layout.T.encode(addr, height, i, hash));
|
||||
b.del(layout.x.encode(addr, hash));
|
||||
}
|
||||
|
||||
if (!tx.isCoinbase()) {
|
||||
for (const {prevout} of tx.inputs) {
|
||||
const {hash, index} = prevout;
|
||||
const coin = view.getOutput(prevout);
|
||||
assert(coin);
|
||||
|
||||
const addr = coin.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
const count = new Count(height, i);
|
||||
|
||||
b.put(layout.C.encode(addr, height, i, hash, index), null);
|
||||
b.put(layout.y.encode(addr, hash, index), count.toRaw());
|
||||
}
|
||||
}
|
||||
|
||||
for (let j = 0; j < tx.outputs.length; j++) {
|
||||
const output = tx.outputs[j];
|
||||
const addr = output.getHash();
|
||||
|
||||
if (!addr)
|
||||
continue;
|
||||
|
||||
b.del(layout.C.encode(addr, height, i, hash, j));
|
||||
b.del(layout.y.encode(addr, hash, j));
|
||||
b.del(layout.A.encode(addr, height, i, hash));
|
||||
b.del(layout.a.encode(addr, hash));
|
||||
}
|
||||
}
|
||||
|
||||
return b.write();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all coins pertinent to an address.
|
||||
* @param {Address} addr
|
||||
* @param {Object} options
|
||||
* @param {Boolean} options.reverse
|
||||
* @param {Boolean} options.limit
|
||||
* @returns {Promise} - Returns {@link Coin}[].
|
||||
*/
|
||||
|
||||
async getCoinsByAddress(addr, options = {}) {
|
||||
const coins = [];
|
||||
|
||||
const {reverse} = options;
|
||||
let {limit} = options;
|
||||
|
||||
if (!limit)
|
||||
limit = this.maxCoins;
|
||||
|
||||
if (limit > this.maxCoins)
|
||||
throw new Error('Limit above max of ${this.maxCoins}.');
|
||||
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
const keys = await this.db.keys({
|
||||
gte: layout.C.min(hash),
|
||||
lte: layout.C.max(hash),
|
||||
limit,
|
||||
reverse,
|
||||
parse: (key) => {
|
||||
const [,,, txid, index] = layout.C.decode(key);
|
||||
return [txid, index];
|
||||
}
|
||||
});
|
||||
|
||||
for (const [hash, index] of keys) {
|
||||
const coin = await this.chain.getCoin(hash, index);
|
||||
assert(coin);
|
||||
coins.push(coin);
|
||||
}
|
||||
|
||||
return coins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all coins pertinent to an address after a
|
||||
* specific txid and output/coin index.
|
||||
* @param {Address} addr
|
||||
* @param {Object} options
|
||||
* @param {Buffer} options.txid
|
||||
* @param {Number} options.index
|
||||
* @param {Boolean} options.limit
|
||||
* @param {Boolean} options.reverse
|
||||
* @returns {Promise} - Returns {@link Coin}[].
|
||||
*/
|
||||
|
||||
async getCoinsByAddressAfter(addr, options = {}) {
|
||||
const coins = [];
|
||||
|
||||
const {txid, index, reverse} = options;
|
||||
let {limit} = options;
|
||||
|
||||
if (!limit)
|
||||
limit = this.maxCoins;
|
||||
|
||||
if (limit > this.maxCoins)
|
||||
throw new Error('Limit above max of ${this.maxCoins}.');
|
||||
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
const raw = await this.db.get(layout.y.encode(hash, txid, index));
|
||||
|
||||
if (!raw)
|
||||
return coins;
|
||||
|
||||
const count = Count.fromRaw(raw);
|
||||
|
||||
const opts = {
|
||||
limit,
|
||||
reverse,
|
||||
parse: (key) => {
|
||||
const [,,, txid, index] = layout.C.decode(key);
|
||||
return [txid, index];
|
||||
}
|
||||
};
|
||||
|
||||
if (!reverse) {
|
||||
opts.gt = layout.C.min(hash, count.height, count.index, txid, count.coin);
|
||||
opts.lte = layout.C.max(hash);
|
||||
} else {
|
||||
opts.gte = layout.C.min(hash);
|
||||
opts.lt = layout.C.max(hash, count.height, count.index, txid, count.coin);
|
||||
}
|
||||
|
||||
const keys = await this.db.keys(opts);
|
||||
|
||||
for (const [hash, index] of keys) {
|
||||
const coin = await this.chain.getCoin(hash, index);
|
||||
assert(coin);
|
||||
coins.push(coin);
|
||||
}
|
||||
|
||||
return coins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all transaction hashes to an address.
|
||||
* @param {Address} addr
|
||||
@ -362,12 +192,12 @@ class AddrIndexer extends Indexer {
|
||||
const hash = Address.getHash(addr);
|
||||
|
||||
await this.db.keys({
|
||||
gte: layout.T.min(hash),
|
||||
lte: layout.T.max(hash),
|
||||
gte: layout.A.min(hash),
|
||||
lte: layout.A.max(hash),
|
||||
limit,
|
||||
reverse,
|
||||
parse: (key) => {
|
||||
const [,,, txid] = layout.T.decode(key);
|
||||
const [,,, txid] = layout.A.decode(key);
|
||||
set.add(txid);
|
||||
}
|
||||
});
|
||||
@ -400,7 +230,7 @@ class AddrIndexer extends Indexer {
|
||||
if (limit > this.maxTxs)
|
||||
throw new Error('Limit above max of ${this.maxTxs}.');
|
||||
|
||||
const raw = await this.db.get(layout.x.encode(hash, txid));
|
||||
const raw = await this.db.get(layout.a.encode(hash, txid));
|
||||
|
||||
if (!raw)
|
||||
return [];
|
||||
@ -412,17 +242,17 @@ class AddrIndexer extends Indexer {
|
||||
limit,
|
||||
reverse,
|
||||
parse: (key) => {
|
||||
const [,,, txid] = layout.T.decode(key);
|
||||
const [,,, txid] = layout.A.decode(key);
|
||||
set.add(txid);
|
||||
}
|
||||
};
|
||||
|
||||
if (!reverse) {
|
||||
opts.gt = layout.T.min(hash, height, index, txid);
|
||||
opts.lte = layout.T.max(hash);
|
||||
opts.gt = layout.A.min(hash, height, index, txid);
|
||||
opts.lte = layout.A.max(hash);
|
||||
} else {
|
||||
opts.gte = layout.T.min(hash);
|
||||
opts.lt = layout.T.max(hash, height, index, txid);
|
||||
opts.gte = layout.A.min(hash);
|
||||
opts.lt = layout.A.max(hash, height, index, txid);
|
||||
}
|
||||
|
||||
await this.db.keys(opts);
|
||||
|
||||
@ -451,37 +451,6 @@ class FullNode extends Node {
|
||||
return this.chain.getCoin(hash, index);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coins that pertain to an address from the mempool or chain database.
|
||||
* Takes into account spent coins in the mempool.
|
||||
* @param {Address} addr
|
||||
* @returns {Promise} - Returns {@link Coin}[].
|
||||
*/
|
||||
|
||||
async getCoinsByAddress(addr) {
|
||||
const mempool = this.mempool.getCoinsByAddress(addr);
|
||||
|
||||
if (!this.addrindex)
|
||||
return mempool;
|
||||
|
||||
const index = await this.addrindex.getCoinsByAddress(addr);
|
||||
const out = [];
|
||||
|
||||
for (const coin of index) {
|
||||
const spent = this.mempool.isSpent(coin.hash, coin.index);
|
||||
|
||||
if (spent)
|
||||
continue;
|
||||
|
||||
out.push(coin);
|
||||
}
|
||||
|
||||
for (const coin of mempool)
|
||||
out.push(coin);
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve transactions pertaining to an
|
||||
* address from the mempool or chain database.
|
||||
|
||||
@ -149,24 +149,6 @@ class HTTP extends Server {
|
||||
});
|
||||
});
|
||||
|
||||
// UTXO by address
|
||||
this.get('/coin/address/:address', async (req, res) => {
|
||||
const valid = Validator.fromRequest(req);
|
||||
const address = valid.str('address');
|
||||
|
||||
enforce(address, 'Address is required.');
|
||||
enforce(!this.chain.options.spv, 'Cannot get coins in SPV mode.');
|
||||
|
||||
const addr = Address.fromString(address, this.network);
|
||||
const coins = await this.node.getCoinsByAddress(addr);
|
||||
const result = [];
|
||||
|
||||
for (const coin of coins)
|
||||
result.push(coin.getJSON(this.network));
|
||||
|
||||
res.json(200, result);
|
||||
});
|
||||
|
||||
// UTXO by id
|
||||
this.get('/coin/:hash/:index', async (req, res) => {
|
||||
const valid = Validator.fromRequest(req);
|
||||
@ -187,23 +169,6 @@ class HTTP extends Server {
|
||||
res.json(200, coin.getJSON(this.network));
|
||||
});
|
||||
|
||||
// Bulk read UTXOs
|
||||
this.post('/coin/address', async (req, res) => {
|
||||
const valid = Validator.fromRequest(req);
|
||||
const address = valid.array('addresses');
|
||||
|
||||
enforce(address, 'Address is required.');
|
||||
enforce(!this.chain.options.spv, 'Cannot get coins in SPV mode.');
|
||||
|
||||
const coins = await this.node.getCoinsByAddress(address);
|
||||
const result = [];
|
||||
|
||||
for (const coin of coins)
|
||||
result.push(coin.getJSON(this.network));
|
||||
|
||||
res.json(200, result);
|
||||
});
|
||||
|
||||
// TX by hash
|
||||
this.get('/tx/:hash', async (req, res) => {
|
||||
const valid = Validator.fromRequest(req);
|
||||
@ -244,25 +209,6 @@ class HTTP extends Server {
|
||||
res.json(200, result);
|
||||
});
|
||||
|
||||
// Bulk read TXs
|
||||
this.post('/tx/address', async (req, res) => {
|
||||
const valid = Validator.fromRequest(req);
|
||||
const address = valid.array('addresses');
|
||||
|
||||
enforce(address, 'Address is required.');
|
||||
enforce(!this.chain.options.spv, 'Cannot get TX in SPV mode.');
|
||||
|
||||
const metas = await this.node.getMetaByAddress(address);
|
||||
const result = [];
|
||||
|
||||
for (const meta of metas) {
|
||||
const view = await this.node.getMetaView(meta);
|
||||
result.push(meta.getJSON(this.network, view, this.chain.height));
|
||||
}
|
||||
|
||||
res.json(200, result);
|
||||
});
|
||||
|
||||
// Block by hash/height
|
||||
this.get('/block/:block', async (req, res) => {
|
||||
const valid = Validator.fromRequest(req);
|
||||
|
||||
@ -92,48 +92,11 @@ describe('Indexer', function() {
|
||||
assert.strictEqual(addrindexer.state.startHeight, 10);
|
||||
});
|
||||
|
||||
it('should get coins by address', async () => {
|
||||
const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 10);
|
||||
});
|
||||
|
||||
it('should get txs by address', async () => {
|
||||
const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
|
||||
assert.strictEqual(hashes.length, 10);
|
||||
});
|
||||
|
||||
it('should get txs for coins by address', async () => {
|
||||
const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 10);
|
||||
|
||||
for (const coin of coins) {
|
||||
const meta = await txindexer.getMeta(coin.hash);
|
||||
assert.bufferEqual(meta.tx.hash(), coin.hash);
|
||||
}
|
||||
});
|
||||
|
||||
it('should coins by address (limit)', async () => {
|
||||
const addr = miner.getAddress();
|
||||
const coins = await addrindexer.getCoinsByAddress(addr, {limit: 1});
|
||||
assert.strictEqual(coins.length, 1);
|
||||
});
|
||||
|
||||
it('should coins by address (reverse)', async () => {
|
||||
const addr = miner.getAddress();
|
||||
const coins = await addrindexer.getCoinsByAddress(
|
||||
addr, {reverse: false});
|
||||
|
||||
assert.strictEqual(coins.length, 10);
|
||||
|
||||
const reversed = await addrindexer.getCoinsByAddress(
|
||||
addr, {reverse: true});
|
||||
|
||||
assert.strictEqual(reversed.length, 10);
|
||||
|
||||
for (let i = 0; i < 10; i++)
|
||||
assert.deepEqual(coins[i], reversed[9 - i]);
|
||||
});
|
||||
|
||||
it('should get txs by address (limit)', async () => {
|
||||
const addr = miner.getAddress();
|
||||
const hashes = await addrindexer.getHashesByAddress(addr, {limit: 1});
|
||||
@ -156,47 +119,6 @@ describe('Indexer', function() {
|
||||
assert.deepEqual(hashes[i], reversed[9 - i]);
|
||||
});
|
||||
|
||||
it('should coins by address after txid and index', async () => {
|
||||
const addr = miner.getAddress();
|
||||
const coins = await addrindexer.getCoinsByAddress(addr, {limit: 5});
|
||||
|
||||
assert.strictEqual(coins.length, 5);
|
||||
|
||||
const txid = coins[4].hash;
|
||||
const index = coins[4].index;
|
||||
|
||||
const next = await addrindexer.getCoinsByAddressAfter(
|
||||
addr, {txid: txid, index: index, limit: 5});
|
||||
|
||||
assert.strictEqual(next.length, 5);
|
||||
|
||||
const all = await addrindexer.getCoinsByAddress(addr);
|
||||
assert.strictEqual(all.length, 10);
|
||||
|
||||
assert.deepEqual(coins.concat(next), all);
|
||||
});
|
||||
|
||||
it('should coins by address after txid and index (reverse)', async () => {
|
||||
const addr = miner.getAddress();
|
||||
const coins = await addrindexer.getCoinsByAddress(
|
||||
addr, {limit: 5, reverse: true});
|
||||
|
||||
assert.strictEqual(coins.length, 5);
|
||||
|
||||
const txid = coins[4].hash;
|
||||
const index = coins[4].index;
|
||||
|
||||
const next = await addrindexer.getCoinsByAddressAfter(
|
||||
addr, {txid: txid, index: index, limit: 5, reverse: true});
|
||||
|
||||
assert.strictEqual(next.length, 5);
|
||||
|
||||
const all = await addrindexer.getCoinsByAddress(addr, {reverse: true});
|
||||
assert.strictEqual(all.length, 10);
|
||||
|
||||
assert.deepEqual(coins.concat(next), all);
|
||||
});
|
||||
|
||||
it('should txs by address after txid', async () => {
|
||||
const addr = miner.getAddress();
|
||||
const hashes = await addrindexer.getHashesByAddress(addr, {limit: 5});
|
||||
@ -251,12 +173,12 @@ describe('Indexer', function() {
|
||||
assert.strictEqual(txindexer.state.startHeight, 20);
|
||||
assert.strictEqual(addrindexer.state.startHeight, 20);
|
||||
|
||||
const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 20);
|
||||
const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
|
||||
assert.strictEqual(hashes.length, 20);
|
||||
|
||||
for (const coin of coins) {
|
||||
const meta = await txindexer.getMeta(coin.hash);
|
||||
assert.bufferEqual(meta.tx.hash(), coin.hash);
|
||||
for (const hash of hashes) {
|
||||
const meta = await txindexer.getMeta(hash);
|
||||
assert.bufferEqual(meta.tx.hash(), hash);
|
||||
}
|
||||
});
|
||||
|
||||
@ -266,12 +188,12 @@ describe('Indexer', function() {
|
||||
assert.strictEqual(txindexer.state.startHeight, 31);
|
||||
assert.strictEqual(addrindexer.state.startHeight, 31);
|
||||
|
||||
const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
|
||||
assert.strictEqual(coins.length, 31);
|
||||
const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
|
||||
assert.strictEqual(hashes.length, 31);
|
||||
|
||||
for (const coin of coins) {
|
||||
const meta = await txindexer.getMeta(coin.hash);
|
||||
assert.bufferEqual(meta.tx.hash(), coin.hash);
|
||||
for (const hash of hashes) {
|
||||
const meta = await txindexer.getMeta(hash);
|
||||
assert.bufferEqual(meta.tx.hash(), hash);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@ -775,38 +775,6 @@ describe('Node', function() {
|
||||
assert.strictEqual(meta.tx.txid(), tx2.txid());
|
||||
});
|
||||
|
||||
it('should get coin/tx by addr', async () => {
|
||||
const addr = await wallet.receiveAddress();
|
||||
const mtx = await wallet.createTX({
|
||||
rate: 100000,
|
||||
outputs: [{
|
||||
value: 100000,
|
||||
address: addr
|
||||
}]
|
||||
});
|
||||
|
||||
await wallet.sign(mtx);
|
||||
|
||||
const tx = mtx.toTX();
|
||||
const job = await miner.createJob();
|
||||
|
||||
job.addTX(tx, mtx.view);
|
||||
job.refresh();
|
||||
|
||||
const block = await job.mineAsync();
|
||||
await chain.add(block);
|
||||
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
|
||||
const txs = await node.getTXByAddress(addr.hash);
|
||||
const tx2 = txs[0];
|
||||
assert.strictEqual(tx.txid(), tx2.txid());
|
||||
|
||||
const coins = await node.getCoinsByAddress(addr.hash);
|
||||
const coin = coins[0];
|
||||
assert.strictEqual(tx.txid(), coin.txid());
|
||||
});
|
||||
|
||||
it('should cleanup', async () => {
|
||||
consensus.COINBASE_MATURITY = 100;
|
||||
await node.close();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user