From 8ecf6e1c471b21231c37545298b002dd4a8f85d7 Mon Sep 17 00:00:00 2001 From: Patrick Nagurny Date: Tue, 17 Jan 2017 15:44:56 -0500 Subject: [PATCH] modify address index, add timestamp and tx indexes (wip) --- lib/services/address/constants.js | 8 +- lib/services/address/encoding.js | 108 ++++++++++++++++++++++++++ lib/services/address/index.js | 3 +- lib/services/db.js | 75 ++++++++++++++++-- lib/services/timestamp.js | 124 ++++++++++++++++++++++++++++++ lib/services/transaction.js | 75 ++++++++++++++++++ 6 files changed, 384 insertions(+), 9 deletions(-) create mode 100644 lib/services/timestamp.js create mode 100644 lib/services/transaction.js diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 3653e9cb..c48407cf 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -3,9 +3,11 @@ var exports = {}; exports.PREFIXES = { - OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height - SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height - SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output + ADDRESS: new Buffer('02', 'hex'), + UNSPENT: new Buffer('03', 'hex'), + // OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height + // SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height + // SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output }; exports.MEMPREFIXES = { diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 8ebce51c..9c9b0483 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -9,6 +9,114 @@ var $ = bitcore.util.preconditions; var exports = {}; +exports.encodeAddressIndexKey = function(hashTypeBuffer, hashBuffer, height, txidBuffer, index, spending) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var indexBuffer = new Buffer(4); + indexBuffer.writeUInt32BE(index); + var spendingBuffer = new Buffer(1); + spendingBuffer.writeUInt8(spending); + + var key = Buffer.concat({ + constants.PREFIXES.ADDRESS, + hashTypeBuffer, + hashBuffer, + constants.SPACER_MIN, + heightBuffer, + txidBuffer, + indexBuffer, + spendingBuffer + }); +}; + +exports.decodeAddressIndexKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashTypeBuffer = reader.read(1); + var hashBuffer = reader.read(20); + + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var txid = reader.read(32); + var index = reader.readUInt32BE(); + var spending = reader.readUInt8(); + return { + prefix: prefix, + hashTypeBuffer: hashTypeBuffer, + hashBuffer: hashBuffer, + height: height, + txid: txid, + index: outputIndex, + spending: spending + }; +}; + +exports.encodeAddressIndexValue = function(satoshis) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return satoshisBuffer; +}; + +exports.decodeAddressIndexValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + return satoshis; +}; + +exports.encodeUnspentIndexKey = function(hashTypeBuffer, hashBuffer, txidBuffer, index) { + var indexBuffer = new Buffer(4); + indexBuffer.writeUInt32BE(index); + + var key = Buffer.concat({ + constants.PREFIXES.UNSPENT, + hashTypeBuffer, + hashBuffer, + constants.SPACER_MIN, + txidBuffer, + indexBuffer + }); +}; + +exports.decodeUnspentIndexKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashTypeBuffer = reader.read(1); + var hashBuffer = reader.read(20); + + var spacer = reader.read(1); + var txid = reader.read(32); + var index = reader.readUInt32BE(); + return { + prefix: prefix, + hashTypeBuffer: hashTypeBuffer, + hashBuffer: hashBuffer, + txid: txid, + index: outputIndex + }; +}; + +exports.encodeUnspentIndexValue = function(satoshis, height, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + + return Buffer.concat([satoshisBuffer, heightBuffer, scriptBuffer]); +}; + +exports.decodeUnspentIndexValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var height = buffer.readUInt32BE(8); + var scriptBuffer = buffer.slice(12, buffer.length); + + return { + satoshis: satoshis, + height: height, + scriptBuffer: scriptBuffer + }; +}; + + exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 2896a178..392cdf18 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -64,7 +64,8 @@ inherits(AddressService, BaseService); AddressService.dependencies = [ 'bitcoind', - 'db' + 'db', + 'transaction' ]; AddressService.prototype.start = function(callback) { diff --git a/lib/services/db.js b/lib/services/db.js index 54496782..8b9c5372 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -42,6 +42,7 @@ function DB(options) { // to determine during an upgrade if a reindex is required this.version = 2; + this.dbPrefix = 0; this.tip = null; this.genesis = null; @@ -70,11 +71,10 @@ util.inherits(DB, Service); DB.dependencies = ['bitcoind']; -DB.PREFIXES = { - VERSION: new Buffer('ff', 'hex'), - BLOCKS: new Buffer('01', 'hex'), - TIP: new Buffer('04', 'hex') -}; +// keys +// 0version +// 0prefix-service +// 0tip // The maximum number of transactions to query at once // Used for populating previous inputs @@ -582,6 +582,7 @@ DB.prototype.runAllBlockHandlers = function(block, add, callback) { ); }; + DB.prototype._encodeBlockIndexKey = function(timestamp) { $.checkArgument(timestamp >= 0 && timestamp <= 4294967295, 'timestamp out of bounds'); var timestampBuffer = new Buffer(4); @@ -801,4 +802,68 @@ DB.prototype.sync = function() { }; +DB.prototype.getPrefix = function(service, callback) { + var self = this; + + async.waterfall( + [ + getPrefix, + getUnused, + putPrefix, + putUnused + ], + callback + ); + + + function getPrefix(next) { + self.store.get(this.dbPrefix + 'prefix-' + service, function(err, buffer) { + if(err && !err.notFound) { + return next(err); + } + + if(!buffer || err.notFound) { + return next(); + } + + // we already have the prefix, call the callback + return callback(null, buffer.readUInt16BE()); + }; + } + + function getUnused(next) { + self.store.get(self.dbPrefix + 'nextUnused', function(err, buffer) { + if(err && !err.notFound) { + return next(err); + } + + if(!buffer || err.notFound) { + return next(null, 1); + } + + return next(null, buffer.readUInt16BE()); + } + } + + function putPrefix(prefix, next) { + self.store.put(self.dbPrefix + 'prefix-' + service, prefix, function(err) { + if(err) { + return next(err); + } + + next(null, prefix); + } + } + + function putUnused(prefix, next) { + self.store.put(self.dbPrefix + 'nextUnused', prefix + 1, function(err) { + if(err) { + return next(err); + } + + return next(null, prefix); + }); + } +}; + module.exports = DB; diff --git a/lib/services/timestamp.js b/lib/services/timestamp.js new file mode 100644 index 00000000..ed245187 --- /dev/null +++ b/lib/services/timestamp.js @@ -0,0 +1,124 @@ +'use strict'; +var BaseService = require('../../service'); +var inherits = require('util').inherits; + +function TimestampService(options) { + BaseService.call(this, options); + +} + +inherits(TimestampService, BaseService); + +TimestampService.dependencies = [ + 'db' +]; + +TimestampService.prototype.start = function(callback) { + var self = this; + + this.store = this.node.services.db.store; + + this.node.services.db.getPrefix(this.name, function(err, prefix) { + if(err) { + return callback(err); + } + + self.prefix = prefix; + + callback(); + }); +}; + +TimestampService.prototype.stop = function(callback) { + setImmediate(callback); +}; + +TimestampService.prototype.blockHandler = function(block, connectBlock, callback) { + var self = this; + + var action = 'put'; + if (!connectBlock) { + action = 'del'; + } + + var operations = []; + + function getLastTimestamp(next) { + var timestamp; + + if(!block.header.prevHash) { + // Genesis block + return next(null, 0); + } + + self.getTimestamp(block.header.prevHash.reverse().toString('hex'), next); + } + + getLastTimestamp(function(err, lastTimestamp) { + if(err) { + return callback(err); + } + + var timestamp = block.header.timestamp; + if(timestamp <= lastTimestamp) { + timestamp = lastTimestamp + 1; + } + + operations.push({ + type: action, + key: self._encodeTimestampBlockKey(timestamp), + value: self._encodeTimestampBlockValue(block.header.hash) + }); + + callback(null, operations); + }); +}; + +Timestamp.prototype.getTimestamp = function(hash, callback) { + var key = this._encodeBlockTimestampKey(hash); + this.store.get(key, function(err, buffer) { + if(err) { + return callback(err); + } + + return callback(null, this._decodeBlockTimestampValue(buffer)); + }); +}; + +TimestampService.prototype._encodeBlockTimestampKey = function(hash) { + return Buffer.concat([self.prefix, new Buffer(hash, 'hex')]); +}; + +TimestampService.prototype._decodeBlockTimestampKey = function(buffer) { + return buffer.slice(1).toString('hex'); +}; + +TimestampService.prototype._encodeBlockTimestampValue = function(timestamp) { + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(timestamp); + return timestampBuffer; +}; + +TimestampService.prototype._decodeBlockTimestampValue = function(buffer) { + return buffer.readDoubleBE(0); +}; + +TimestampService.prototype._encodeTimestampBlockKey = function(timestamp) { + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(timestamp); + return Buffer.concat([self.prefix, timestampBuffer]); +}; + +TimestampService.prototype._decodeTimestampBlockKey = function(buffer) { + return buffer.readDoubleBE(1); +}; + +TimestampService.prototype._encodeTimestampBlockValue = function(hash) { + return new Buffer(hash, 'hex'); +}; + +TimestampService.prototype._decodeTimestampBlockValue = function(buffer) { + return buffer.toString('hex'); +}; + +module.exports = TimestampService; \ No newline at end of file diff --git a/lib/services/transaction.js b/lib/services/transaction.js new file mode 100644 index 00000000..2dc6ea44 --- /dev/null +++ b/lib/services/transaction.js @@ -0,0 +1,75 @@ +'use strict'; +var BaseService = require('../../service'); +var inherits = require('util').inherits; + +function TransactionService(options) { + BaseService.call(this, options); + +} + +inherits(TransactionService, BaseService); + +TransactionService.dependencies = [ + 'db' +]; + +TransactionService.prototype.start = function(callback) { + var self = this; + + this.store = this.node.services.db.store; + + this.node.services.db.getPrefix(this.name, function(err, prefix) { + if(err) { + return callback(err); + } + + self.prefix = prefix; + + callback(); + }); +}; + +TransactionService.prototype.stop = function(callback) { + setImmediate(callback); +}; + +TransactionService.prototype.blockHandler = function(block, connectBlock, callback) { + var action = 'put'; + if (!connectBlock) { + action = 'del'; + } + + var operations = []; + + for(var i = 0; i < block.transactions.length; i++) { + var tx = block.transactions[i]; + + operations.push({ + type: action, + key: this._encodeTransactionKey(tx.id), + value: this._encodeTransactionValue(tx) + }); + } + + setImmediate(function() { + callback(null, operations); + }); +}; + +TransactionService.prototype._encodeTransactionKey = function(txid) { + return Buffer.concat([this.prefix, new Buffer(txid, 'hex')]); +}; + +TransactionService.prototype._decodeTransactionKey = function(buffer) { + return buffer.slice(1).toString('hex'); +}; + +TransactionService.prototype._encodeTransactionValue = function(transaction) { + return transaction.uncheckedSerialize(); +}; + +TransactionService.prototype._decodeTransactionValue = function(buffer) { + return new bitcore.Transaction(buffer); +}; + +module.exports = TransactionService; \ No newline at end of file