Added new index for spent txs.

This commit is contained in:
Chris Kleeschulte 2017-10-18 19:03:04 -04:00
parent 5a5dbb624c
commit 98be272925
No known key found for this signature in database
GPG Key ID: 33195D27EF6BDB7F
7 changed files with 330 additions and 91 deletions

65
: Normal file
View File

@ -0,0 +1,65 @@
'use strict';
var should = require('chai').should();
var Tx = require('bcoin').tx;
var Encoding = require('../../../lib/services/transaction/encoding');
describe('Transaction service encoding', function() {
var servicePrefix = new Buffer('0000', 'hex');
var encoding = new Encoding(servicePrefix);
var txid = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add';
var blockHash = txid;
var txHex = '0100000001cc3ffe0638792c8b39328bb490caaefe2cf418f2ce0144956e0c22515f29724d010000006a473044022030ce9fa68d1a32abf0cd4adecf90fb998375b64fe887c6987278452b068ae74c022036a7d00d1c8af19e298e04f14294c807ebda51a20389ad751b4ff3c032cf8990012103acfcb348abb526526a9f63214639d79183871311c05b2eebc727adfdd016514fffffffff02f6ae7d04000000001976a9144455183e407ee4d3423858c8a3275918aedcd18e88aca99b9b08010000001976a9140beceae2c29bfde08d2b6d80b33067451c5887be88ac00000000';
var tx = Tx.fromRaw(txHex, 'hex');
var txEncoded = Buffer.concat([new Buffer('00000002', 'hex'), new Buffer('00000001', 'hex'), new Buffer('0002', 'hex'), new Buffer('40000000000000004008000000000000', 'hex'), tx.toRaw()]);
var indexBuf = new Buffer(4);
indexBuf.writeUInt32BE(3);
it('should encode transaction key' , function() {
var txBuf = new Buffer(txid, 'hex');
encoding.encodeTransactionKey(txid).should.deep.equal(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), txBuf]));
});
it('should decode transaction key', function() {
encoding.decodeTransactionKey(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), new Buffer(txid, 'hex')]))
.should.equal(txid);
});
it('should encode transaction value', function() {
tx.__height = 2;
tx.__blockHash = blockHash;
tx.__timestamp = 1;
tx.__inputValues = [ 2, 3 ];
encoding.encodeTransactionValue(tx).should.deep.equal(txEncoded);
});
it('should decode transaction value', function() {
var tx = encoding.decodeTransactionValue(txEncoded);
tx.__height.should.equal(2);
tx.__timestamp.should.equal(1);
tx.__inputValues.should.deep.equal([2,3]);
tx.toRaw().toString('hex').should.equal(txHex);
});
it('should encode spent key', function() {
encoding.encodeSpentKey(txid, 3).should.deep.equal(Buffer.concat([servicePrefix,
new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf]));
});
it('should decode spent key', function() {
encoding.decodeSpentKey(Buffer.concat([servicePrefix,
new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf])).should.deep.equal({ txid: txid, outputIndex: 3 });
});
it('should encode spent value', function() {
encoding.encodeSpentValue(txid, 3).should.deep.equal(Buffer.concat([new Buffer(txid, 'hex'), indexBuf]));
});
it('should decode spent value', function() {
encoding.decodeSpentValue(Buffer.concat([new Buffer(txid, 'hex'), indexBuf])).should.deep.equal({ txid: txid, inputIndex: 3 });
});
});

View File

@ -44,78 +44,6 @@ AddressService.dependencies = [
'mempool'
];
AddressService.prototype._applySpendingTxsToOutputs = function(tx, options, callback) {
var self = this;
async.eachOfLimit(tx.outputs, 4, function(output, index, next) {
var address = utils.getAddress(output, self._network);
if (!address || !tx.__height) {
return next();
}
self.getAddressHistory(address, { start: tx.__height }, function(err, history) {
if (err) {
return next(err);
}
// there should be zero or one tx count, if there are more, then this output was double spent
if (history.totalCount === 1) {
var spentTx = history.items[0];
tx.outputs[index].spentTxId = spentTx.txid();
for(var i = 0; i < spentTx.inputs.length; i++) {
var input = spentTx.inputs[i];
if (input.prevout.txid() === tx.txid()) {
output.spentIndex = i;
}
}
assert(output.spentIndex >= 0, 'Transaction Service: found spending tx, but could not find index.');
output.spentHeight = tx.__height || 0;
}
next();
});
}, function(err) {
if (err) {
return callback(err);
}
console.log(tx.outputs[0].spentTxId);
callback(null, tx);
});
};
AddressService.prototype.getTransactionWithAddressInfo = function(txid, options, callback) {
var self = this;
self._transaction.getTransaction(txid, options, function(err, tx) {
if (err) {
return callback(err);
}
if (!tx) {
return callback();
}
// locate any tx that spent this tx's outputs
self._applySpendingTxsToOutputs(tx, options, callback);
});
};
AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
var self = this;
@ -147,7 +75,7 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
txList = utils.orderByConfirmations(txList);
var results = {
totalCount: options.txCount,
totalCount: options.txCount || 0,
items: txList
};
@ -205,6 +133,7 @@ AddressService.prototype.getAddressSummary = function(address, options, callback
AddressService.prototype._getAddressSummaryResult = function(txs, address, result, options) {
var self = this;
for(var i = 0; i < txs.length; i++) {
var tx = txs[i];

View File

@ -78,7 +78,14 @@ DB.prototype.start = function(callback) {
mkdirp.sync(this.dataPath);
}
this._store = levelup(this.dataPath, { db: this.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary'});
this._store = levelup(this.dataPath, {
db: this.levelupStore,
keyEncoding: 'binary',
valueEncoding: 'binary',
writeBufferSize: 8 * 1024 * 1024,
maxOpenFiles: 3000,
cacheSize: 1024 * 1024 * 1024 // 1 GB of memory for cache.
});
setImmediate(callback);

View File

@ -4,21 +4,25 @@ var Tx = require('bcoin').tx;
function Encoding(servicePrefix) {
this.servicePrefix = servicePrefix;
this.txIndex = new Buffer('00', 'hex');
this.spentIndex = new Buffer('01', 'hex');
this.doubleSpentIndex = new Buffer('02', 'hex');
}
Encoding.prototype.encodeTransactionKey = function(txid) {
return Buffer.concat([this.servicePrefix, new Buffer(txid, 'hex')]);
return Buffer.concat([this.servicePrefix, this.txIndex, new Buffer(txid, 'hex')]);
};
Encoding.prototype.decodeTransactionKey = function(buffer) {
return buffer.slice(2).toString('hex');
return buffer.slice(3).toString('hex');
};
// TODO: maybe we should be storing the block hash here too.
Encoding.prototype.encodeTransactionValue = function(transaction) {
var heightBuffer = new Buffer(4);
heightBuffer.writeUInt32BE(transaction.__height);
var hashBuffer = new Buffer(transaction.__blockhash, 'hex');
var timestampBuffer = new Buffer(4);
timestampBuffer.writeUInt32BE(transaction.__timestamp);
@ -31,28 +35,108 @@ Encoding.prototype.encodeTransactionValue = function(transaction) {
var inputValuesLengthBuffer = new Buffer(2);
inputValuesLengthBuffer.writeUInt16BE(inputValues.length);
return new Buffer.concat([heightBuffer, timestampBuffer,
return new Buffer.concat([heightBuffer, hashBuffer, timestampBuffer,
inputValuesLengthBuffer, inputValuesBuffer, transaction.toRaw()]);
};
Encoding.prototype.decodeTransactionValue = function(buffer) {
var height = buffer.readUInt32BE();
var timestamp = buffer.readUInt32BE(4);
var inputValuesLength = buffer.readUInt16BE(8);
var blockhash = buffer.slice(4, 36).toString('hex');
var timestamp = buffer.readUInt32BE(36);
var inputValuesLength = buffer.readUInt16BE(40);
var inputValues = [];
for(var i = 0; i < inputValuesLength; i++) {
inputValues.push(buffer.readDoubleBE(i * 8 + 10));
inputValues.push(buffer.readDoubleBE(i * 8 + 42));
}
var txBuf = buffer.slice(inputValues.length * 8 + 10);
var txBuf = buffer.slice(inputValues.length * 8 + 42);
var transaction = Tx.fromRaw(txBuf);
transaction.__height = height;
transaction.__blockhash = blockhash;
transaction.__inputValues = inputValues;
transaction.__timestamp = timestamp;
return transaction;
};
// for every input we receive, we make an entry for what output it spends
Encoding.prototype.encodeSpentKey = function(txid, outputIndex) {
var outputIndexBuffer = new Buffer(4);
outputIndexBuffer.writeUInt32BE(outputIndex);
return Buffer.concat([this.servicePrefix, this.spentIndex, new Buffer(txid, 'hex'), outputIndexBuffer]);
};
Encoding.prototype.decodeSpentKey = function(buffer) {
var txid = buffer.slice(3, 35).toString('hex');
var outputIndex = buffer.readUInt32BE(35);
return {
txid: txid,
outputIndex: outputIndex
};
};
Encoding.prototype.encodeSpentValue = function(txid, inputIndex, blockHeight, blockHash) {
var inputIndexBuffer = new Buffer(4);
inputIndexBuffer.writeUInt32BE(inputIndex);
var blockHeightBuffer = new Buffer(4);
blockHeightBuffer.writeUInt32BE(blockHeight);
var blockHashBuffer = new Buffer(blockHash, 'hex');
return Buffer.concat([new Buffer(txid, 'hex'), inputIndexBuffer, blockHeightBuffer, blockHashBuffer]);
};
Encoding.prototype.decodeSpentValue = function(buffer) {
var txid = buffer.slice(0, 32).toString('hex');
var inputIndex = buffer.readUInt32BE(32);
var blockHeight = buffer.readUInt32BE(36, 40);
var blockHash = buffer.slice(40).toString('hex');
return {
txid: txid,
inputIndex: inputIndex,
blockHeight: blockHeight,
blockHash: blockHash
};
};
Encoding.prototype.encodeDoubleSpentKey = function(txid, outputIndex) {
var outputIndexBuffer = new Buffer(4);
outputIndexBuffer.writeUInt32BE(outputIndex);
return Buffer.concat([this.servicePrefix, this.spentIndex, new Buffer(txid, 'hex'), outputIndexBuffer]);
};
Encoding.prototype.decodeDoubleSpentKey = function(buffer) {
var txid = buffer.slice(3, 35).toString('hex');
var outputIndex = buffer.readUInt32BE(35);
return {
txid: txid,
outputIndex: outputIndex
};
};
Encoding.prototype.encodeDoubleSpentValue = function(txid, inputIndex, blockHeight, blockHash) {
var inputIndexBuffer = new Buffer(4);
inputIndexBuffer.writeUInt32BE(inputIndex);
var blockHeightBuffer = new Buffer(4);
blockHeightBuffer.writeUInt32BE(inputIndex);
var blockHashBuffer = new Buffer(blockHash, 'hex');
return Buffer.concat([new Buffer(txid, 'hex'), inputIndexBuffer, blockHeightBuffer, blockHashBuffer]);
};
Encoding.prototype.decodeDoubleSpentValue = function(buffer) {
var txid = buffer.slice(0, 32).toString('hex');
var inputIndex = buffer.readUInt32BE(32, 36);
var blockHeight = buffer.readUInt32BE(36, 40);
var blockHash = buffer.slice(40).toString('hex');
return {
txid: txid,
inputIndex: inputIndex,
blockHeight: blockHeight,
blockHash: blockHash
};
};
module.exports = Encoding;

View File

@ -51,6 +51,65 @@ TransactionService.prototype.getAPIMethods = function() {
];
};
TransactionService.prototype.getDetailedTransaction = function(txid, options, callback) {
var self = this;
self.getTransaction(txid, options, function(err, tx) {
if (err) {
return callback(err);
}
if (!tx) {
return callback();
}
// get the spentTxId, spentHeight, spentIndex, spendBlockHash
async.parallel([
function(next) {
async.eachOfLimit(tx.outputs, 4, function(output, index, next) {
self._db.get(self._encoding.encodeSpentKey(txid, index), function(err, value) {
if (err) {
return next(err);
}
if (!value) {
return next();
}
var spentIndex = self._encoding.decodeSpentValue(value);
tx.outputs[index].spentTxId = spentIndex.txid;
tx.outputs[index].spentIndex = spentIndex.inputIndex;
tx.outputs[index].spentHeight = spentIndex.blockHeight;
tx.outputs[index].spentBlockHash = spentIndex.blockHash;
next();
});
}, next);
},
function(next) {
async.eachOfLimit(tx.inputs, 4, function(input, index, next) {
self._db.get(self._encoding.encodeDoubleSpentKey(input.prevout.txid(), index), function(err, value) {
if (err) {
return next(err);
}
if (!value) {
return next();
}
var doubleSpendInfo = self._encoding.decodeDoubleSpentValue(value);
tx.inputs[index].doubleSpentTxID = doubleSpendInfo.txid;
next();
});
}, next);
}
], function(err) {
if (err) {
return callback(err);
}
callback(null, tx);
});
});
};
TransactionService.prototype.getTransaction = function(txid, options, callback) {
var self = this;
@ -341,7 +400,7 @@ TransactionService.prototype.onBlock = function(block, callback) {
assert(block.txs.length === operations.length, 'It seems we are not indexing the correct number of transactions.');
callback(null, operations);
callback(null, _.flattenDeep(operations));
});
};
@ -354,6 +413,8 @@ TransactionService.prototype.onReorg = function(args, callback) {
var removalOps = [];
// remove the txid -> tx entries
// remove the prevTxid, outputIndex -> txid, inputIndex
for(var i = 0; i < oldBlockList.length; i++) {
var block = oldBlockList[i];
@ -367,6 +428,15 @@ TransactionService.prototype.onReorg = function(args, callback) {
key: self._encoding.encodeTransactionKey(tx.txid())
});
// remove all the spent index information
for(j = 0; j < tx.inputs.length; j++) {
var input = tx.inputs[j];
removalOps.push({
type: 'del',
key: self._encoding.encodeSpentKey(input.prevout.txid(), input.prevout.index)
});
}
}
}
@ -376,6 +446,47 @@ TransactionService.prototype.onReorg = function(args, callback) {
};
TransactionService.prototype._getSpentInfo = function(input, callback) {
this._db.get(this._encoding.encodeSpentKey(input.prevout.txid(), input.prevout.index), callback);
};
TransactionService.prototype._getSpentTxOperations = function(tx, callback) {
var self = this;
var ops = [];
// if any of this tx's inputs are double spending, then make an entry into this index.
async.eachOfLimit(tx.inputs, 4, function(input, index, next) {
self._getSpentInfo(input, function(err, info) {
if (err) {
return callback(err);
}
if (info) {
ops.push({
key: self._encoding.encodeDoubleSpentKey(input.prevout.txid(), input.prevout.index),
value: self._encoding.encodeDoubleSpentValue(tx.txid(), index, tx.__height, tx.__blockhash)
});
return next();
}
ops.push({
key: self._encoding.encodeSpentKey(input.prevout.txid(), input.prevout.index),
value: self._encoding.encodeSpentValue(tx.txid(), index, tx.__height, tx.__blockhash)
});
next();
});
}, function(err) {
if (err) {
return callback(err);
}
callback(null, ops);
});
};
TransactionService.prototype._processTransaction = function(tx, opts, callback) {
// this index is very simple txid -> tx, but we also need to find each
@ -404,10 +515,27 @@ TransactionService.prototype._processTransaction = function(tx, opts, callback)
tx.__height = opts.block.__height;
assert(tx.__height, 'Block height is required when saving a trasnaction.');
callback(null, {
// block hash
tx.__blockhash = opts.block.rhash();
var operations = [{
key: self._encoding.encodeTransactionKey(tx.txid()),
value: self._encoding.encodeTransactionValue(tx)
}];
// spent key and value
// for each input in this tx, it spend some tx's prev outs, so index those
// this also accounts for double spend operations
self._getSpentTxOperations(tx, function(err, ops) {
if (err) {
return callback(err);
}
operations = operations.concat(ops);
callback(null, operations);
});
});
};

View File

@ -82,6 +82,9 @@ describe('Address Service', function() {
var getHeaderHash = sandbox.stub().callsArgWith(1, null, 'aa');
var getBlockHeader = sandbox.stub().callsArgWith(1, null, 'aa');
var getTxsByAddress = sandbox.stub().callsArgWith(1, null, []);
var getTransaction = sandbox.stub().callsArgWith(2, null, { __height: 123, outputs: [ { value: 1 } ], __inputValues: [ 1 ] });
addressService._transaction = { getTransaction: getTransaction };
addressService._mempool = { getTxsByAddress: getTxsByAddress };
addressService._header = {
@ -89,11 +92,9 @@ describe('Address Service', function() {
getBlockHeader: getBlockHeader
};
var address = 'a';
var opts = { from: 12, to: 14 };
var opts = { from: 0, to: 10 };
var txid = '1c6ea4a55a3edaac0a05e93b52908f607376a8fdc5387c492042f8baa6c05085';
var data = [ null, encoding.encodeAddressIndexKey(address, 123, txid, 1, 1) ];
var getTransaction = sandbox.stub().callsArgWith(2, null, { __height: 123, outputs: [ { value: 1 } ], __inputValues: [ 1 ] });
addressService._tx = { getTransaction: getTransaction };
var txidStream = new Readable();

View File

@ -1,5 +1,4 @@
'use strict';
var should = require('chai').should();
var Tx = require('bcoin').tx;
@ -8,24 +7,29 @@ var Encoding = require('../../../lib/services/transaction/encoding');
describe('Transaction service encoding', function() {
var servicePrefix = new Buffer('0000', 'hex');
var encoding = new Encoding(servicePrefix);
var txid = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add';
var blockHash = txid;
var txHex = '0100000001cc3ffe0638792c8b39328bb490caaefe2cf418f2ce0144956e0c22515f29724d010000006a473044022030ce9fa68d1a32abf0cd4adecf90fb998375b64fe887c6987278452b068ae74c022036a7d00d1c8af19e298e04f14294c807ebda51a20389ad751b4ff3c032cf8990012103acfcb348abb526526a9f63214639d79183871311c05b2eebc727adfdd016514fffffffff02f6ae7d04000000001976a9144455183e407ee4d3423858c8a3275918aedcd18e88aca99b9b08010000001976a9140beceae2c29bfde08d2b6d80b33067451c5887be88ac00000000';
var tx = Tx.fromRaw(txHex, 'hex');
var txEncoded = Buffer.concat([new Buffer('00000002', 'hex'), new Buffer('00000001', 'hex'), new Buffer('0002', 'hex'), new Buffer('40000000000000004008000000000000', 'hex'), tx.toRaw()]);
var txEncoded = Buffer.concat([new Buffer('00000002', 'hex'), new Buffer(blockHash, 'hex'), new Buffer('00000001', 'hex'), new Buffer('0002', 'hex'), new Buffer('40000000000000004008000000000000', 'hex'), tx.toRaw()]);
var indexBuf = new Buffer(4);
indexBuf.writeUInt32BE(3);
it('should encode transaction key' , function() {
var txBuf = new Buffer(txid, 'hex');
encoding.encodeTransactionKey(txid).should.deep.equal(Buffer.concat([servicePrefix, txBuf]));
encoding.encodeTransactionKey(txid).should.deep.equal(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), txBuf]));
});
it('should decode transaction key', function() {
encoding.decodeTransactionKey(Buffer.concat([servicePrefix, new Buffer(txid, 'hex')]))
encoding.decodeTransactionKey(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), new Buffer(txid, 'hex')]))
.should.equal(txid);
});
it('should encode transaction value', function() {
tx.__height = 2;
tx.__blockhash = blockHash;
tx.__timestamp = 1;
tx.__inputValues = [ 2, 3 ];
@ -35,8 +39,29 @@ describe('Transaction service encoding', function() {
it('should decode transaction value', function() {
var tx = encoding.decodeTransactionValue(txEncoded);
tx.__height.should.equal(2);
tx.__timestamp.should.equal(1);
tx.__inputValues.should.deep.equal([2,3]);
tx.toRaw().toString('hex').should.equal(txHex);
});
it('should encode spent key', function() {
encoding.encodeSpentKey(txid, 3).should.deep.equal(Buffer.concat([servicePrefix,
new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf]));
});
it('should decode spent key', function() {
encoding.decodeSpentKey(Buffer.concat([servicePrefix,
new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf])).should.deep.equal({ txid: txid, outputIndex: 3 });
});
it('should encode spent value', function() {
encoding.encodeSpentValue(txid, 3, 3, txid).should.deep.equal(Buffer.concat([new Buffer(txid, 'hex'), indexBuf, indexBuf, new Buffer(blockHash, 'hex')]));
});
it('should decode spent value', function() {
encoding.decodeSpentValue(Buffer.concat([new Buffer(txid, 'hex'), indexBuf,
indexBuf, new Buffer(blockHash, 'hex')]))
.should.deep.equal({ txid: txid, inputIndex: 3, blockHeight: 3, blockHash: blockHash });
});
});