From 98be272925bdeb2bb1d206ccabc239c9d7aa94b3 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 18 Oct 2017 19:03:04 -0400 Subject: [PATCH] Added new index for spent txs. --- : | 65 ++++++++++ lib/services/address/index.js | 75 +----------- lib/services/db/index.js | 9 +- lib/services/transaction/encoding.js | 100 ++++++++++++++-- lib/services/transaction/index.js | 132 ++++++++++++++++++++- test/services/address/index.unit.js | 7 +- test/services/transaction/encoding.unit.js | 33 +++++- 7 files changed, 330 insertions(+), 91 deletions(-) create mode 100644 : diff --git a/: b/: new file mode 100644 index 00000000..7febdb6b --- /dev/null +++ b/: @@ -0,0 +1,65 @@ +'use strict'; +var should = require('chai').should(); +var Tx = require('bcoin').tx; + +var Encoding = require('../../../lib/services/transaction/encoding'); + +describe('Transaction service encoding', function() { + + var servicePrefix = new Buffer('0000', 'hex'); + + var encoding = new Encoding(servicePrefix); + var txid = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add'; + var blockHash = txid; + var txHex = '0100000001cc3ffe0638792c8b39328bb490caaefe2cf418f2ce0144956e0c22515f29724d010000006a473044022030ce9fa68d1a32abf0cd4adecf90fb998375b64fe887c6987278452b068ae74c022036a7d00d1c8af19e298e04f14294c807ebda51a20389ad751b4ff3c032cf8990012103acfcb348abb526526a9f63214639d79183871311c05b2eebc727adfdd016514fffffffff02f6ae7d04000000001976a9144455183e407ee4d3423858c8a3275918aedcd18e88aca99b9b08010000001976a9140beceae2c29bfde08d2b6d80b33067451c5887be88ac00000000'; + var tx = Tx.fromRaw(txHex, 'hex'); + var txEncoded = Buffer.concat([new Buffer('00000002', 'hex'), new Buffer('00000001', 'hex'), new Buffer('0002', 'hex'), new Buffer('40000000000000004008000000000000', 'hex'), tx.toRaw()]); + var indexBuf = new Buffer(4); + indexBuf.writeUInt32BE(3); + + it('should encode transaction key' , function() { + var txBuf = new Buffer(txid, 'hex'); + encoding.encodeTransactionKey(txid).should.deep.equal(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), txBuf])); + }); + + it('should decode transaction key', function() { + encoding.decodeTransactionKey(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), new Buffer(txid, 'hex')])) + .should.equal(txid); + }); + + it('should encode transaction value', function() { + tx.__height = 2; + tx.__blockHash = blockHash; + tx.__timestamp = 1; + tx.__inputValues = [ 2, 3 ]; + + encoding.encodeTransactionValue(tx).should.deep.equal(txEncoded); + }); + + it('should decode transaction value', function() { + var tx = encoding.decodeTransactionValue(txEncoded); + tx.__height.should.equal(2); + + tx.__timestamp.should.equal(1); + tx.__inputValues.should.deep.equal([2,3]); + tx.toRaw().toString('hex').should.equal(txHex); + }); + + it('should encode spent key', function() { + encoding.encodeSpentKey(txid, 3).should.deep.equal(Buffer.concat([servicePrefix, + new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf])); + }); + + it('should decode spent key', function() { + encoding.decodeSpentKey(Buffer.concat([servicePrefix, + new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf])).should.deep.equal({ txid: txid, outputIndex: 3 }); + }); + + it('should encode spent value', function() { + encoding.encodeSpentValue(txid, 3).should.deep.equal(Buffer.concat([new Buffer(txid, 'hex'), indexBuf])); + }); + + it('should decode spent value', function() { + encoding.decodeSpentValue(Buffer.concat([new Buffer(txid, 'hex'), indexBuf])).should.deep.equal({ txid: txid, inputIndex: 3 }); + }); +}); diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 94369bf1..3765921b 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -44,78 +44,6 @@ AddressService.dependencies = [ 'mempool' ]; -AddressService.prototype._applySpendingTxsToOutputs = function(tx, options, callback) { - - var self = this; - - async.eachOfLimit(tx.outputs, 4, function(output, index, next) { - - var address = utils.getAddress(output, self._network); - - if (!address || !tx.__height) { - return next(); - } - - self.getAddressHistory(address, { start: tx.__height }, function(err, history) { - - if (err) { - return next(err); - } - - // there should be zero or one tx count, if there are more, then this output was double spent - if (history.totalCount === 1) { - var spentTx = history.items[0]; - - tx.outputs[index].spentTxId = spentTx.txid(); - - for(var i = 0; i < spentTx.inputs.length; i++) { - var input = spentTx.inputs[i]; - if (input.prevout.txid() === tx.txid()) { - output.spentIndex = i; - } - } - - assert(output.spentIndex >= 0, 'Transaction Service: found spending tx, but could not find index.'); - - output.spentHeight = tx.__height || 0; - } - next(); - - }); - - }, function(err) { - - if (err) { - return callback(err); - } - -console.log(tx.outputs[0].spentTxId); - callback(null, tx); - - }); - -}; - -AddressService.prototype.getTransactionWithAddressInfo = function(txid, options, callback) { - - var self = this; - self._transaction.getTransaction(txid, options, function(err, tx) { - - if (err) { - return callback(err); - } - - if (!tx) { - return callback(); - } - - // locate any tx that spent this tx's outputs - self._applySpendingTxsToOutputs(tx, options, callback); - - }); - -}; - AddressService.prototype.getAddressHistory = function(addresses, options, callback) { var self = this; @@ -147,7 +75,7 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba txList = utils.orderByConfirmations(txList); var results = { - totalCount: options.txCount, + totalCount: options.txCount || 0, items: txList }; @@ -205,6 +133,7 @@ AddressService.prototype.getAddressSummary = function(address, options, callback AddressService.prototype._getAddressSummaryResult = function(txs, address, result, options) { var self = this; + for(var i = 0; i < txs.length; i++) { var tx = txs[i]; diff --git a/lib/services/db/index.js b/lib/services/db/index.js index f44c6276..cdaccf89 100644 --- a/lib/services/db/index.js +++ b/lib/services/db/index.js @@ -78,7 +78,14 @@ DB.prototype.start = function(callback) { mkdirp.sync(this.dataPath); } - this._store = levelup(this.dataPath, { db: this.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary'}); + this._store = levelup(this.dataPath, { + db: this.levelupStore, + keyEncoding: 'binary', + valueEncoding: 'binary', + writeBufferSize: 8 * 1024 * 1024, + maxOpenFiles: 3000, + cacheSize: 1024 * 1024 * 1024 // 1 GB of memory for cache. + }); setImmediate(callback); diff --git a/lib/services/transaction/encoding.js b/lib/services/transaction/encoding.js index 39238272..add531e8 100644 --- a/lib/services/transaction/encoding.js +++ b/lib/services/transaction/encoding.js @@ -4,21 +4,25 @@ var Tx = require('bcoin').tx; function Encoding(servicePrefix) { this.servicePrefix = servicePrefix; + this.txIndex = new Buffer('00', 'hex'); + this.spentIndex = new Buffer('01', 'hex'); + this.doubleSpentIndex = new Buffer('02', 'hex'); } Encoding.prototype.encodeTransactionKey = function(txid) { - return Buffer.concat([this.servicePrefix, new Buffer(txid, 'hex')]); + return Buffer.concat([this.servicePrefix, this.txIndex, new Buffer(txid, 'hex')]); }; Encoding.prototype.decodeTransactionKey = function(buffer) { - return buffer.slice(2).toString('hex'); + return buffer.slice(3).toString('hex'); }; -// TODO: maybe we should be storing the block hash here too. Encoding.prototype.encodeTransactionValue = function(transaction) { var heightBuffer = new Buffer(4); heightBuffer.writeUInt32BE(transaction.__height); + var hashBuffer = new Buffer(transaction.__blockhash, 'hex'); + var timestampBuffer = new Buffer(4); timestampBuffer.writeUInt32BE(transaction.__timestamp); @@ -31,28 +35,108 @@ Encoding.prototype.encodeTransactionValue = function(transaction) { var inputValuesLengthBuffer = new Buffer(2); inputValuesLengthBuffer.writeUInt16BE(inputValues.length); - return new Buffer.concat([heightBuffer, timestampBuffer, + return new Buffer.concat([heightBuffer, hashBuffer, timestampBuffer, inputValuesLengthBuffer, inputValuesBuffer, transaction.toRaw()]); }; Encoding.prototype.decodeTransactionValue = function(buffer) { var height = buffer.readUInt32BE(); - var timestamp = buffer.readUInt32BE(4); - var inputValuesLength = buffer.readUInt16BE(8); + var blockhash = buffer.slice(4, 36).toString('hex'); + + var timestamp = buffer.readUInt32BE(36); + + var inputValuesLength = buffer.readUInt16BE(40); + var inputValues = []; for(var i = 0; i < inputValuesLength; i++) { - inputValues.push(buffer.readDoubleBE(i * 8 + 10)); + inputValues.push(buffer.readDoubleBE(i * 8 + 42)); } - var txBuf = buffer.slice(inputValues.length * 8 + 10); + var txBuf = buffer.slice(inputValues.length * 8 + 42); var transaction = Tx.fromRaw(txBuf); transaction.__height = height; + transaction.__blockhash = blockhash; transaction.__inputValues = inputValues; transaction.__timestamp = timestamp; return transaction; }; +// for every input we receive, we make an entry for what output it spends +Encoding.prototype.encodeSpentKey = function(txid, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([this.servicePrefix, this.spentIndex, new Buffer(txid, 'hex'), outputIndexBuffer]); +}; + +Encoding.prototype.decodeSpentKey = function(buffer) { + var txid = buffer.slice(3, 35).toString('hex'); + var outputIndex = buffer.readUInt32BE(35); + return { + txid: txid, + outputIndex: outputIndex + }; +}; + +Encoding.prototype.encodeSpentValue = function(txid, inputIndex, blockHeight, blockHash) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + var blockHeightBuffer = new Buffer(4); + blockHeightBuffer.writeUInt32BE(blockHeight); + var blockHashBuffer = new Buffer(blockHash, 'hex'); + return Buffer.concat([new Buffer(txid, 'hex'), inputIndexBuffer, blockHeightBuffer, blockHashBuffer]); +}; + +Encoding.prototype.decodeSpentValue = function(buffer) { + var txid = buffer.slice(0, 32).toString('hex'); + var inputIndex = buffer.readUInt32BE(32); + var blockHeight = buffer.readUInt32BE(36, 40); + var blockHash = buffer.slice(40).toString('hex'); + return { + txid: txid, + inputIndex: inputIndex, + blockHeight: blockHeight, + blockHash: blockHash + }; +}; + +Encoding.prototype.encodeDoubleSpentKey = function(txid, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([this.servicePrefix, this.spentIndex, new Buffer(txid, 'hex'), outputIndexBuffer]); +}; + +Encoding.prototype.decodeDoubleSpentKey = function(buffer) { + var txid = buffer.slice(3, 35).toString('hex'); + var outputIndex = buffer.readUInt32BE(35); + return { + txid: txid, + outputIndex: outputIndex + }; +}; + +Encoding.prototype.encodeDoubleSpentValue = function(txid, inputIndex, blockHeight, blockHash) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + var blockHeightBuffer = new Buffer(4); + blockHeightBuffer.writeUInt32BE(inputIndex); + var blockHashBuffer = new Buffer(blockHash, 'hex'); + return Buffer.concat([new Buffer(txid, 'hex'), inputIndexBuffer, blockHeightBuffer, blockHashBuffer]); +}; + +Encoding.prototype.decodeDoubleSpentValue = function(buffer) { + var txid = buffer.slice(0, 32).toString('hex'); + var inputIndex = buffer.readUInt32BE(32, 36); + var blockHeight = buffer.readUInt32BE(36, 40); + var blockHash = buffer.slice(40).toString('hex'); + return { + txid: txid, + inputIndex: inputIndex, + blockHeight: blockHeight, + blockHash: blockHash + }; +}; + module.exports = Encoding; diff --git a/lib/services/transaction/index.js b/lib/services/transaction/index.js index a72a4c5a..a36e4591 100644 --- a/lib/services/transaction/index.js +++ b/lib/services/transaction/index.js @@ -51,6 +51,65 @@ TransactionService.prototype.getAPIMethods = function() { ]; }; +TransactionService.prototype.getDetailedTransaction = function(txid, options, callback) { + + var self = this; + self.getTransaction(txid, options, function(err, tx) { + + if (err) { + return callback(err); + } + + if (!tx) { + return callback(); + } + + // get the spentTxId, spentHeight, spentIndex, spendBlockHash + async.parallel([ + function(next) { + async.eachOfLimit(tx.outputs, 4, function(output, index, next) { + self._db.get(self._encoding.encodeSpentKey(txid, index), function(err, value) { + if (err) { + return next(err); + } + if (!value) { + return next(); + } + var spentIndex = self._encoding.decodeSpentValue(value); + tx.outputs[index].spentTxId = spentIndex.txid; + tx.outputs[index].spentIndex = spentIndex.inputIndex; + tx.outputs[index].spentHeight = spentIndex.blockHeight; + tx.outputs[index].spentBlockHash = spentIndex.blockHash; + next(); + }); + }, next); + }, + function(next) { + async.eachOfLimit(tx.inputs, 4, function(input, index, next) { + self._db.get(self._encoding.encodeDoubleSpentKey(input.prevout.txid(), index), function(err, value) { + if (err) { + return next(err); + } + if (!value) { + return next(); + } + var doubleSpendInfo = self._encoding.decodeDoubleSpentValue(value); + tx.inputs[index].doubleSpentTxID = doubleSpendInfo.txid; + next(); + }); + }, next); + } + ], function(err) { + if (err) { + return callback(err); + } + callback(null, tx); + }); + }); + +}; + + TransactionService.prototype.getTransaction = function(txid, options, callback) { var self = this; @@ -341,7 +400,7 @@ TransactionService.prototype.onBlock = function(block, callback) { assert(block.txs.length === operations.length, 'It seems we are not indexing the correct number of transactions.'); - callback(null, operations); + callback(null, _.flattenDeep(operations)); }); }; @@ -354,6 +413,8 @@ TransactionService.prototype.onReorg = function(args, callback) { var removalOps = []; + // remove the txid -> tx entries + // remove the prevTxid, outputIndex -> txid, inputIndex for(var i = 0; i < oldBlockList.length; i++) { var block = oldBlockList[i]; @@ -367,6 +428,15 @@ TransactionService.prototype.onReorg = function(args, callback) { key: self._encoding.encodeTransactionKey(tx.txid()) }); + // remove all the spent index information + for(j = 0; j < tx.inputs.length; j++) { + var input = tx.inputs[j]; + removalOps.push({ + type: 'del', + key: self._encoding.encodeSpentKey(input.prevout.txid(), input.prevout.index) + }); + } + } } @@ -376,6 +446,47 @@ TransactionService.prototype.onReorg = function(args, callback) { }; +TransactionService.prototype._getSpentInfo = function(input, callback) { + this._db.get(this._encoding.encodeSpentKey(input.prevout.txid(), input.prevout.index), callback); +}; + +TransactionService.prototype._getSpentTxOperations = function(tx, callback) { + var self = this; + + var ops = []; + // if any of this tx's inputs are double spending, then make an entry into this index. + async.eachOfLimit(tx.inputs, 4, function(input, index, next) { + + self._getSpentInfo(input, function(err, info) { + + if (err) { + return callback(err); + } + + if (info) { + ops.push({ + key: self._encoding.encodeDoubleSpentKey(input.prevout.txid(), input.prevout.index), + value: self._encoding.encodeDoubleSpentValue(tx.txid(), index, tx.__height, tx.__blockhash) + }); + return next(); + } + + ops.push({ + key: self._encoding.encodeSpentKey(input.prevout.txid(), input.prevout.index), + value: self._encoding.encodeSpentValue(tx.txid(), index, tx.__height, tx.__blockhash) + }); + next(); + + }); + }, function(err) { + if (err) { + return callback(err); + } + callback(null, ops); + + }); +}; + TransactionService.prototype._processTransaction = function(tx, opts, callback) { // this index is very simple txid -> tx, but we also need to find each @@ -404,10 +515,27 @@ TransactionService.prototype._processTransaction = function(tx, opts, callback) tx.__height = opts.block.__height; assert(tx.__height, 'Block height is required when saving a trasnaction.'); - callback(null, { + // block hash + tx.__blockhash = opts.block.rhash(); + var operations = [{ key: self._encoding.encodeTransactionKey(tx.txid()), value: self._encoding.encodeTransactionValue(tx) + }]; + + // spent key and value + // for each input in this tx, it spend some tx's prev outs, so index those + // this also accounts for double spend operations + self._getSpentTxOperations(tx, function(err, ops) { + + if (err) { + return callback(err); + } + + operations = operations.concat(ops); + callback(null, operations); + }); + }); }; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 4507ed3c..646afbd1 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -82,6 +82,9 @@ describe('Address Service', function() { var getHeaderHash = sandbox.stub().callsArgWith(1, null, 'aa'); var getBlockHeader = sandbox.stub().callsArgWith(1, null, 'aa'); var getTxsByAddress = sandbox.stub().callsArgWith(1, null, []); + var getTransaction = sandbox.stub().callsArgWith(2, null, { __height: 123, outputs: [ { value: 1 } ], __inputValues: [ 1 ] }); + + addressService._transaction = { getTransaction: getTransaction }; addressService._mempool = { getTxsByAddress: getTxsByAddress }; addressService._header = { @@ -89,11 +92,9 @@ describe('Address Service', function() { getBlockHeader: getBlockHeader }; var address = 'a'; - var opts = { from: 12, to: 14 }; + var opts = { from: 0, to: 10 }; var txid = '1c6ea4a55a3edaac0a05e93b52908f607376a8fdc5387c492042f8baa6c05085'; var data = [ null, encoding.encodeAddressIndexKey(address, 123, txid, 1, 1) ]; - var getTransaction = sandbox.stub().callsArgWith(2, null, { __height: 123, outputs: [ { value: 1 } ], __inputValues: [ 1 ] }); - addressService._tx = { getTransaction: getTransaction }; var txidStream = new Readable(); diff --git a/test/services/transaction/encoding.unit.js b/test/services/transaction/encoding.unit.js index ef66dd78..3b8c28db 100644 --- a/test/services/transaction/encoding.unit.js +++ b/test/services/transaction/encoding.unit.js @@ -1,5 +1,4 @@ 'use strict'; - var should = require('chai').should(); var Tx = require('bcoin').tx; @@ -8,24 +7,29 @@ var Encoding = require('../../../lib/services/transaction/encoding'); describe('Transaction service encoding', function() { var servicePrefix = new Buffer('0000', 'hex'); + var encoding = new Encoding(servicePrefix); var txid = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add'; + var blockHash = txid; var txHex = '0100000001cc3ffe0638792c8b39328bb490caaefe2cf418f2ce0144956e0c22515f29724d010000006a473044022030ce9fa68d1a32abf0cd4adecf90fb998375b64fe887c6987278452b068ae74c022036a7d00d1c8af19e298e04f14294c807ebda51a20389ad751b4ff3c032cf8990012103acfcb348abb526526a9f63214639d79183871311c05b2eebc727adfdd016514fffffffff02f6ae7d04000000001976a9144455183e407ee4d3423858c8a3275918aedcd18e88aca99b9b08010000001976a9140beceae2c29bfde08d2b6d80b33067451c5887be88ac00000000'; var tx = Tx.fromRaw(txHex, 'hex'); - var txEncoded = Buffer.concat([new Buffer('00000002', 'hex'), new Buffer('00000001', 'hex'), new Buffer('0002', 'hex'), new Buffer('40000000000000004008000000000000', 'hex'), tx.toRaw()]); + var txEncoded = Buffer.concat([new Buffer('00000002', 'hex'), new Buffer(blockHash, 'hex'), new Buffer('00000001', 'hex'), new Buffer('0002', 'hex'), new Buffer('40000000000000004008000000000000', 'hex'), tx.toRaw()]); + var indexBuf = new Buffer(4); + indexBuf.writeUInt32BE(3); it('should encode transaction key' , function() { var txBuf = new Buffer(txid, 'hex'); - encoding.encodeTransactionKey(txid).should.deep.equal(Buffer.concat([servicePrefix, txBuf])); + encoding.encodeTransactionKey(txid).should.deep.equal(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), txBuf])); }); it('should decode transaction key', function() { - encoding.decodeTransactionKey(Buffer.concat([servicePrefix, new Buffer(txid, 'hex')])) + encoding.decodeTransactionKey(Buffer.concat([servicePrefix, new Buffer('00', 'hex'), new Buffer(txid, 'hex')])) .should.equal(txid); }); it('should encode transaction value', function() { tx.__height = 2; + tx.__blockhash = blockHash; tx.__timestamp = 1; tx.__inputValues = [ 2, 3 ]; @@ -35,8 +39,29 @@ describe('Transaction service encoding', function() { it('should decode transaction value', function() { var tx = encoding.decodeTransactionValue(txEncoded); tx.__height.should.equal(2); + tx.__timestamp.should.equal(1); tx.__inputValues.should.deep.equal([2,3]); tx.toRaw().toString('hex').should.equal(txHex); }); + + it('should encode spent key', function() { + encoding.encodeSpentKey(txid, 3).should.deep.equal(Buffer.concat([servicePrefix, + new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf])); + }); + + it('should decode spent key', function() { + encoding.decodeSpentKey(Buffer.concat([servicePrefix, + new Buffer('01', 'hex'), new Buffer(txid, 'hex'), indexBuf])).should.deep.equal({ txid: txid, outputIndex: 3 }); + }); + + it('should encode spent value', function() { + encoding.encodeSpentValue(txid, 3, 3, txid).should.deep.equal(Buffer.concat([new Buffer(txid, 'hex'), indexBuf, indexBuf, new Buffer(blockHash, 'hex')])); + }); + + it('should decode spent value', function() { + encoding.decodeSpentValue(Buffer.concat([new Buffer(txid, 'hex'), indexBuf, + indexBuf, new Buffer(blockHash, 'hex')])) + .should.deep.equal({ txid: txid, inputIndex: 3, blockHeight: 3, blockHash: blockHash }); + }); });