RanchiMall/fcoin
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

indexer: remove address coin index

Browse Source
This commit is contained in:
Braydon Fuller 2019-03-25 15:30:12 -07:00
parent 05d55efb22
commit 7dc55c9c3c
No known key found for this signature in database
GPG Key ID: F24F232D108B3AD4
5 changed files with 42 additions and 407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
lib/indexer/addrindexer.js
View File

@ -16,17 +16,24 @@ const Indexer = require('./indexer');
/* /*
* AddrIndexer Database Layout: * AddrIndexer Database Layout:
* T[addr-hash][height][tx-index][hash] -> dummy (tx by address) * A[addr-hash][height][index][hash] -> dummy (tx by address, height and index)
* C[addr-hash][height][tx-index][hash][coin-index] -> dummy (coin by address) * a[addr-hash][hash] -> (tx height and index by address and tx hash)
* x[addr-hash][hash] -> height and tx-index for tx *
* y[addr-hash][hash][index] -> height, tx-index and coin-index for coin * The database layout is organized so that transactions are sorted in
*/ * the same order as the blocks (e.g. chronological order) using the block
* height and transaction index. This provides the ability to query for
* sets of transactions within that order. For a wallet that would like to
* synchronize or rescan, this could be a query for all of the latest
* transactions, but not for earlier transactions that are already known.
* Furthermore, to be able to query for all transactions in multiple sets
* without reference to height and index, there is a mapping from address
* and tx hash to the height and index as an entry point to the
* ordered transactions.
*/
Object.assign(layout, { Object.assign(layout, {
T: bdb.key('T', ['hash', 'uint32', 'uint32', 'hash256']), A: bdb.key('A', ['hash', 'uint32', 'uint32', 'hash256']),
C: bdb.key('C', ['hash', 'uint32', 'uint32', 'hash256', 'uint32']), a: bdb.key('a', ['hash', 'hash256'])
x: bdb.key('x', ['hash', 'hash256']),
y: bdb.key('y', ['hash', 'hash256', 'uint32'])
}); });
/** /**
@ -42,15 +49,11 @@ class Count {
*/ */
constructor(height, index, coin) { constructor(height, index, coin) {
this.height = height >= 0 ? height : 0; this.height = height || 0;
this.index = index >= 0 ? index : 0; this.index = index || 0;
this.coin = coin >= 0 ? coin : -1;
assert((this.height >>> 0) === this.height); assert((this.height >>> 0) === this.height);
assert((this.index >>> 0) === this.index); assert((this.index >>> 0) === this.index);
if (coin)
assert((this.coin >>> 0) === this.coin);
} }
/** /**
@ -59,18 +62,11 @@ class Count {
*/ */
toRaw() { toRaw() {
let len = 8; const bw = bio.write(8);
if (this.coin >= 0)
len += 4;
const bw = bio.write(len);
bw.writeU32(this.height); bw.writeU32(this.height);
bw.writeU32(this.index); bw.writeU32(this.index);
if (this.coin >= 0)
bw.writeU32(this.coin);
return bw.render(); return bw.render();
} }
@ -86,9 +82,6 @@ class Count {
this.height = br.readU32(); this.height = br.readU32();
this.index = br.readU32(); this.index = br.readU32();
if (br.left() >= 4)
this.coin = br.readU32();
return this; return this;
} }
@ -121,7 +114,6 @@ class AddrIndexer extends Indexer {
this.db = bdb.create(this.options); this.db = bdb.create(this.options);
this.maxTxs = options.maxTxs || 100; this.maxTxs = options.maxTxs || 100;
this.maxCoins = options.maxCoins || 500;
} }
/** /**
@ -143,37 +135,8 @@ class AddrIndexer extends Indexer {
for (const addr of tx.getHashes(view)) { for (const addr of tx.getHashes(view)) {
const count = new Count(height, i); const count = new Count(height, i);
b.put(layout.T.encode(addr, height, i, hash), null); b.put(layout.A.encode(addr, height, i, hash), null);
b.put(layout.x.encode(addr, hash), count.toRaw()); b.put(layout.a.encode(addr, hash), count.toRaw());
}
if (!tx.isCoinbase()) {
for (const {prevout} of tx.inputs) {
const {hash, index} = prevout;
const coin = view.getOutput(prevout);
assert(coin);
const addr = coin.getHash();
if (!addr)
continue;
b.del(layout.C.encode(addr, height, i, hash, index));
b.del(layout.y.encode(addr, hash, index));
}
}
for (let j = 0; j < tx.outputs.length; j++) {
const output = tx.outputs[j];
const addr = output.getHash();
if (!addr)
continue;
const count = new Count(height, i, j);
b.put(layout.C.encode(addr, height, i, hash, j), null);
b.put(layout.y.encode(addr, hash, j), count.toRaw());
} }
} }
@ -197,147 +160,14 @@ class AddrIndexer extends Indexer {
const hash = tx.hash(); const hash = tx.hash();
for (const addr of tx.getHashes(view)) { for (const addr of tx.getHashes(view)) {
b.del(layout.T.encode(addr, height, i, hash)); b.del(layout.A.encode(addr, height, i, hash));
b.del(layout.x.encode(addr, hash)); b.del(layout.a.encode(addr, hash));
}
if (!tx.isCoinbase()) {
for (const {prevout} of tx.inputs) {
const {hash, index} = prevout;
const coin = view.getOutput(prevout);
assert(coin);
const addr = coin.getHash();
if (!addr)
continue;
const count = new Count(height, i);
b.put(layout.C.encode(addr, height, i, hash, index), null);
b.put(layout.y.encode(addr, hash, index), count.toRaw());
}
}
for (let j = 0; j < tx.outputs.length; j++) {
const output = tx.outputs[j];
const addr = output.getHash();
if (!addr)
continue;
b.del(layout.C.encode(addr, height, i, hash, j));
b.del(layout.y.encode(addr, hash, j));
} }
} }
return b.write(); return b.write();
} }
/**
* Get all coins pertinent to an address.
* @param {Address} addr
* @param {Object} options
* @param {Boolean} options.reverse
* @param {Boolean} options.limit
* @returns {Promise} - Returns {@link Coin}[].
*/
async getCoinsByAddress(addr, options = {}) {
const coins = [];
const {reverse} = options;
let {limit} = options;
if (!limit)
limit = this.maxCoins;
if (limit > this.maxCoins)
throw new Error('Limit above max of ${this.maxCoins}.');
const hash = Address.getHash(addr);
const keys = await this.db.keys({
gte: layout.C.min(hash),
lte: layout.C.max(hash),
limit,
reverse,
parse: (key) => {
const [,,, txid, index] = layout.C.decode(key);
return [txid, index];
}
});
for (const [hash, index] of keys) {
const coin = await this.chain.getCoin(hash, index);
assert(coin);
coins.push(coin);
}
return coins;
}
/**
* Get all coins pertinent to an address after a
* specific txid and output/coin index.
* @param {Address} addr
* @param {Object} options
* @param {Buffer} options.txid
* @param {Number} options.index
* @param {Boolean} options.limit
* @param {Boolean} options.reverse
* @returns {Promise} - Returns {@link Coin}[].
*/
async getCoinsByAddressAfter(addr, options = {}) {
const coins = [];
const {txid, index, reverse} = options;
let {limit} = options;
if (!limit)
limit = this.maxCoins;
if (limit > this.maxCoins)
throw new Error('Limit above max of ${this.maxCoins}.');
const hash = Address.getHash(addr);
const raw = await this.db.get(layout.y.encode(hash, txid, index));
if (!raw)
return coins;
const count = Count.fromRaw(raw);
const opts = {
limit,
reverse,
parse: (key) => {
const [,,, txid, index] = layout.C.decode(key);
return [txid, index];
}
};
if (!reverse) {
opts.gt = layout.C.min(hash, count.height, count.index, txid, count.coin);
opts.lte = layout.C.max(hash);
} else {
opts.gte = layout.C.min(hash);
opts.lt = layout.C.max(hash, count.height, count.index, txid, count.coin);
}
const keys = await this.db.keys(opts);
for (const [hash, index] of keys) {
const coin = await this.chain.getCoin(hash, index);
assert(coin);
coins.push(coin);
}
return coins;
}
/** /**
* Get all transaction hashes to an address. * Get all transaction hashes to an address.
* @param {Address} addr * @param {Address} addr
@ -362,12 +192,12 @@ class AddrIndexer extends Indexer {
const hash = Address.getHash(addr); const hash = Address.getHash(addr);
await this.db.keys({ await this.db.keys({
gte: layout.T.min(hash), gte: layout.A.min(hash),
lte: layout.T.max(hash), lte: layout.A.max(hash),
limit, limit,
reverse, reverse,
parse: (key) => { parse: (key) => {
const [,,, txid] = layout.T.decode(key); const [,,, txid] = layout.A.decode(key);
set.add(txid); set.add(txid);
} }
}); });
@ -400,7 +230,7 @@ class AddrIndexer extends Indexer {
if (limit > this.maxTxs) if (limit > this.maxTxs)
throw new Error('Limit above max of ${this.maxTxs}.'); throw new Error('Limit above max of ${this.maxTxs}.');
const raw = await this.db.get(layout.x.encode(hash, txid)); const raw = await this.db.get(layout.a.encode(hash, txid));
if (!raw) if (!raw)
return []; return [];
@ -412,17 +242,17 @@ class AddrIndexer extends Indexer {
limit, limit,
reverse, reverse,
parse: (key) => { parse: (key) => {
const [,,, txid] = layout.T.decode(key); const [,,, txid] = layout.A.decode(key);
set.add(txid); set.add(txid);
} }
}; };
if (!reverse) { if (!reverse) {
opts.gt = layout.T.min(hash, height, index, txid); opts.gt = layout.A.min(hash, height, index, txid);
opts.lte = layout.T.max(hash); opts.lte = layout.A.max(hash);
} else { } else {
opts.gte = layout.T.min(hash); opts.gte = layout.A.min(hash);
opts.lt = layout.T.max(hash, height, index, txid); opts.lt = layout.A.max(hash, height, index, txid);
} }
await this.db.keys(opts); await this.db.keys(opts);

31
lib/node/fullnode.js
View File

@ -451,37 +451,6 @@ class FullNode extends Node {
return this.chain.getCoin(hash, index); return this.chain.getCoin(hash, index);
} }
/**
* Get coins that pertain to an address from the mempool or chain database.
* Takes into account spent coins in the mempool.
* @param {Address} addr
* @returns {Promise} - Returns {@link Coin}[].
*/
async getCoinsByAddress(addr) {
const mempool = this.mempool.getCoinsByAddress(addr);
if (!this.addrindex)
return mempool;
const index = await this.addrindex.getCoinsByAddress(addr);
const out = [];
for (const coin of index) {
const spent = this.mempool.isSpent(coin.hash, coin.index);
if (spent)
continue;
out.push(coin);
}
for (const coin of mempool)
out.push(coin);
return out;
}
/** /**
* Retrieve transactions pertaining to an * Retrieve transactions pertaining to an
* address from the mempool or chain database. * address from the mempool or chain database.

54
lib/node/http.js
View File

@ -149,24 +149,6 @@ class HTTP extends Server {
}); });
}); });
// UTXO by address
this.get('/coin/address/:address', async (req, res) => {
const valid = Validator.fromRequest(req);
const address = valid.str('address');
enforce(address, 'Address is required.');
enforce(!this.chain.options.spv, 'Cannot get coins in SPV mode.');
const addr = Address.fromString(address, this.network);
const coins = await this.node.getCoinsByAddress(addr);
const result = [];
for (const coin of coins)
result.push(coin.getJSON(this.network));
res.json(200, result);
});
// UTXO by id // UTXO by id
this.get('/coin/:hash/:index', async (req, res) => { this.get('/coin/:hash/:index', async (req, res) => {
const valid = Validator.fromRequest(req); const valid = Validator.fromRequest(req);
@ -187,23 +169,6 @@ class HTTP extends Server {
res.json(200, coin.getJSON(this.network)); res.json(200, coin.getJSON(this.network));
}); });
// Bulk read UTXOs
this.post('/coin/address', async (req, res) => {
const valid = Validator.fromRequest(req);
const address = valid.array('addresses');
enforce(address, 'Address is required.');
enforce(!this.chain.options.spv, 'Cannot get coins in SPV mode.');
const coins = await this.node.getCoinsByAddress(address);
const result = [];
for (const coin of coins)
result.push(coin.getJSON(this.network));
res.json(200, result);
});
// TX by hash // TX by hash
this.get('/tx/:hash', async (req, res) => { this.get('/tx/:hash', async (req, res) => {
const valid = Validator.fromRequest(req); const valid = Validator.fromRequest(req);
@ -244,25 +209,6 @@ class HTTP extends Server {
res.json(200, result); res.json(200, result);
}); });
// Bulk read TXs
this.post('/tx/address', async (req, res) => {
const valid = Validator.fromRequest(req);
const address = valid.array('addresses');
enforce(address, 'Address is required.');
enforce(!this.chain.options.spv, 'Cannot get TX in SPV mode.');
const metas = await this.node.getMetaByAddress(address);
const result = [];
for (const meta of metas) {
const view = await this.node.getMetaView(meta);
result.push(meta.getJSON(this.network, view, this.chain.height));
}
res.json(200, result);
});
// Block by hash/height // Block by hash/height
this.get('/block/:block', async (req, res) => { this.get('/block/:block', async (req, res) => {
const valid = Validator.fromRequest(req); const valid = Validator.fromRequest(req);

98
test/indexer-test.js
View File

@ -92,48 +92,11 @@ describe('Indexer', function() {
assert.strictEqual(addrindexer.state.startHeight, 10); assert.strictEqual(addrindexer.state.startHeight, 10);
}); });
it('should get coins by address', async () => {
const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
assert.strictEqual(coins.length, 10);
});
it('should get txs by address', async () => { it('should get txs by address', async () => {
const hashes = await addrindexer.getHashesByAddress(miner.getAddress()); const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
assert.strictEqual(hashes.length, 10); assert.strictEqual(hashes.length, 10);
}); });
it('should get txs for coins by address', async () => {
const coins = await addrindexer.getCoinsByAddress(miner.getAddress());
assert.strictEqual(coins.length, 10);
for (const coin of coins) {
const meta = await txindexer.getMeta(coin.hash);
assert.bufferEqual(meta.tx.hash(), coin.hash);
}
});
it('should coins by address (limit)', async () => {
const addr = miner.getAddress();
const coins = await addrindexer.getCoinsByAddress(addr, {limit: 1});
assert.strictEqual(coins.length, 1);
});
it('should coins by address (reverse)', async () => {
const addr = miner.getAddress();
const coins = await addrindexer.getCoinsByAddress(
addr, {reverse: false});
assert.strictEqual(coins.length, 10);
const reversed = await addrindexer.getCoinsByAddress(
addr, {reverse: true});
assert.strictEqual(reversed.length, 10);
for (let i = 0; i < 10; i++)
assert.deepEqual(coins[i], reversed[9 - i]);
});
it('should get txs by address (limit)', async () => { it('should get txs by address (limit)', async () => {
const addr = miner.getAddress(); const addr = miner.getAddress();
const hashes = await addrindexer.getHashesByAddress(addr, {limit: 1}); const hashes = await addrindexer.getHashesByAddress(addr, {limit: 1});
@ -156,47 +119,6 @@ describe('Indexer', function() {
assert.deepEqual(hashes[i], reversed[9 - i]); assert.deepEqual(hashes[i], reversed[9 - i]);
}); });
it('should coins by address after txid and index', async () => {
const addr = miner.getAddress();
const coins = await addrindexer.getCoinsByAddress(addr, {limit: 5});
assert.strictEqual(coins.length, 5);
const txid = coins[4].hash;
const index = coins[4].index;
const next = await addrindexer.getCoinsByAddressAfter(
addr, {txid: txid, index: index, limit: 5});
assert.strictEqual(next.length, 5);
const all = await addrindexer.getCoinsByAddress(addr);
assert.strictEqual(all.length, 10);
assert.deepEqual(coins.concat(next), all);
});
it('should coins by address after txid and index (reverse)', async () => {
const addr = miner.getAddress();
const coins = await addrindexer.getCoinsByAddress(
addr, {limit: 5, reverse: true});
assert.strictEqual(coins.length, 5);
const txid = coins[4].hash;
const index = coins[4].index;
const next = await addrindexer.getCoinsByAddressAfter(
addr, {txid: txid, index: index, limit: 5, reverse: true});
assert.strictEqual(next.length, 5);
const all = await addrindexer.getCoinsByAddress(addr, {reverse: true});
assert.strictEqual(all.length, 10);
assert.deepEqual(coins.concat(next), all);
});
it('should txs by address after txid', async () => { it('should txs by address after txid', async () => {
const addr = miner.getAddress(); const addr = miner.getAddress();
const hashes = await addrindexer.getHashesByAddress(addr, {limit: 5}); const hashes = await addrindexer.getHashesByAddress(addr, {limit: 5});
@ -251,12 +173,12 @@ describe('Indexer', function() {
assert.strictEqual(txindexer.state.startHeight, 20); assert.strictEqual(txindexer.state.startHeight, 20);
assert.strictEqual(addrindexer.state.startHeight, 20); assert.strictEqual(addrindexer.state.startHeight, 20);
const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
assert.strictEqual(coins.length, 20); assert.strictEqual(hashes.length, 20);
for (const coin of coins) { for (const hash of hashes) {
const meta = await txindexer.getMeta(coin.hash); const meta = await txindexer.getMeta(hash);
assert.bufferEqual(meta.tx.hash(), coin.hash); assert.bufferEqual(meta.tx.hash(), hash);
} }
}); });
@ -266,12 +188,12 @@ describe('Indexer', function() {
assert.strictEqual(txindexer.state.startHeight, 31); assert.strictEqual(txindexer.state.startHeight, 31);
assert.strictEqual(addrindexer.state.startHeight, 31); assert.strictEqual(addrindexer.state.startHeight, 31);
const coins = await addrindexer.getCoinsByAddress(miner.getAddress()); const hashes = await addrindexer.getHashesByAddress(miner.getAddress());
assert.strictEqual(coins.length, 31); assert.strictEqual(hashes.length, 31);
for (const coin of coins) { for (const hash of hashes) {
const meta = await txindexer.getMeta(coin.hash); const meta = await txindexer.getMeta(hash);
assert.bufferEqual(meta.tx.hash(), coin.hash); assert.bufferEqual(meta.tx.hash(), hash);
} }
}); });
}); });

32
test/node-test.js
View File

@ -775,38 +775,6 @@ describe('Node', function() {
assert.strictEqual(meta.tx.txid(), tx2.txid()); assert.strictEqual(meta.tx.txid(), tx2.txid());
}); });
it('should get coin/tx by addr', async () => {
const addr = await wallet.receiveAddress();
const mtx = await wallet.createTX({
rate: 100000,
outputs: [{
value: 100000,
address: addr
}]
});
await wallet.sign(mtx);
const tx = mtx.toTX();
const job = await miner.createJob();
job.addTX(tx, mtx.view);
job.refresh();
const block = await job.mineAsync();
await chain.add(block);
await new Promise(r => setTimeout(r, 300));
const txs = await node.getTXByAddress(addr.hash);
const tx2 = txs[0];
assert.strictEqual(tx.txid(), tx2.txid());
const coins = await node.getCoinsByAddress(addr.hash);
const coin = coins[0];
assert.strictEqual(tx.txid(), coin.txid());
});
it('should cleanup', async () => { it('should cleanup', async () => {
consensus.COINBASE_MATURITY = 100; consensus.COINBASE_MATURITY = 100;
await node.close(); await node.close();