modify address index, add timestamp and tx indexes (wip)

lib/services/address/constants.js

@@ -3,9 +3,11 @@
 var exports = {};
 
 exports.PREFIXES = {
-  OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
+  ADDRESS: new Buffer('02', 'hex'),
-  SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
+  UNSPENT: new Buffer('03', 'hex'),
-  SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
+  // OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
+  // SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
+  // SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
 };
 
 exports.MEMPREFIXES = {

lib/services/address/encoding.js

@@ -9,6 +9,114 @@ var $ = bitcore.util.preconditions;
 
 var exports = {};
 
+exports.encodeAddressIndexKey = function(hashTypeBuffer, hashBuffer, height, txidBuffer, index, spending) {
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
+  var indexBuffer = new Buffer(4);
+  indexBuffer.writeUInt32BE(index);
+  var spendingBuffer = new Buffer(1);
+  spendingBuffer.writeUInt8(spending);
+
+  var key = Buffer.concat({
+    constants.PREFIXES.ADDRESS,
+    hashTypeBuffer,
+    hashBuffer,
+    constants.SPACER_MIN,
+    heightBuffer,
+    txidBuffer,
+    indexBuffer,
+    spendingBuffer
+  });
+};
+
+exports.decodeAddressIndexKey = function(buffer) {
+  var reader = new BufferReader(buffer);
+  var prefix = reader.read(1);
+  var hashTypeBuffer = reader.read(1);
+  var hashBuffer = reader.read(20);
+
+  var spacer = reader.read(1);
+  var height = reader.readUInt32BE();
+  var txid = reader.read(32);
+  var index = reader.readUInt32BE();
+  var spending = reader.readUInt8();
+  return {
+    prefix: prefix,
+    hashTypeBuffer: hashTypeBuffer,
+    hashBuffer: hashBuffer,
+    height: height,
+    txid: txid,
+    index: outputIndex,
+    spending: spending
+  };
+};
+
+exports.encodeAddressIndexValue = function(satoshis) {
+  var satoshisBuffer = new Buffer(8);
+  satoshisBuffer.writeDoubleBE(satoshis);
+  return satoshisBuffer;
+};
+
+exports.decodeAddressIndexValue = function(buffer) {
+  var satoshis = buffer.readDoubleBE(0);
+  return satoshis;
+};
+
+exports.encodeUnspentIndexKey = function(hashTypeBuffer, hashBuffer, txidBuffer, index) {
+  var indexBuffer = new Buffer(4);
+  indexBuffer.writeUInt32BE(index);
+
+  var key = Buffer.concat({
+    constants.PREFIXES.UNSPENT,
+    hashTypeBuffer,
+    hashBuffer,
+    constants.SPACER_MIN,
+    txidBuffer,
+    indexBuffer
+  });
+};
+
+exports.decodeUnspentIndexKey = function(buffer) {
+  var reader = new BufferReader(buffer);
+  var prefix = reader.read(1);
+  var hashTypeBuffer = reader.read(1);
+  var hashBuffer = reader.read(20);
+
+  var spacer = reader.read(1);
+  var txid = reader.read(32);
+  var index = reader.readUInt32BE();
+  return {
+    prefix: prefix,
+    hashTypeBuffer: hashTypeBuffer,
+    hashBuffer: hashBuffer,
+    txid: txid,
+    index: outputIndex
+  };
+};
+
+exports.encodeUnspentIndexValue = function(satoshis, height, scriptBuffer) {
+  var satoshisBuffer = new Buffer(8);
+  satoshisBuffer.writeDoubleBE(satoshis);
+
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
+
+  return Buffer.concat([satoshisBuffer, heightBuffer, scriptBuffer]);
+};
+
+exports.decodeUnspentIndexValue = function(buffer) {
+  var satoshis = buffer.readDoubleBE(0);
+  var height = buffer.readUInt32BE(8);
+  var scriptBuffer = buffer.slice(12, buffer.length);
+
+  return {
+    satoshis: satoshis,
+    height: height,
+    scriptBuffer: scriptBuffer
+  };
+};
+
+
 exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) {
   var outputIndexBuffer = new Buffer(4);
   outputIndexBuffer.writeUInt32BE(outputIndex);

lib/services/address/index.js

@@ -64,7 +64,8 @@ inherits(AddressService, BaseService);
 
 AddressService.dependencies = [
   'bitcoind',
-  'db'
+  'db',
+  'transaction'
 ];
 
 AddressService.prototype.start = function(callback) {

lib/services/db.js

@@ -42,6 +42,7 @@ function DB(options) {
   // to determine during an upgrade if a reindex is required
   this.version = 2;
 
+  this.dbPrefix = 0;
   this.tip = null;
   this.genesis = null;
 
@@ -70,11 +71,10 @@ util.inherits(DB, Service);
 
 DB.dependencies = ['bitcoind'];
 
-DB.PREFIXES = {
+// keys
-  VERSION: new Buffer('ff', 'hex'),
+// 0version
-  BLOCKS: new Buffer('01', 'hex'),
+// 0prefix-service
-  TIP: new Buffer('04', 'hex')
+// 0tip
-};
 
 // The maximum number of transactions to query at once
 // Used for populating previous inputs
@@ -582,6 +582,7 @@ DB.prototype.runAllBlockHandlers = function(block, add, callback) {
   );
 };
 
+
 DB.prototype._encodeBlockIndexKey = function(timestamp) {
   $.checkArgument(timestamp >= 0 && timestamp <= 4294967295, 'timestamp out of bounds');
   var timestampBuffer = new Buffer(4);
@@ -801,4 +802,68 @@ DB.prototype.sync = function() {
 
 };
 
+DB.prototype.getPrefix = function(service, callback) {
+  var self = this;
+
+  async.waterfall(
+    [
+      getPrefix,
+      getUnused,
+      putPrefix,
+      putUnused
+    ],
+    callback
+  );
+
+
+  function getPrefix(next) {
+    self.store.get(this.dbPrefix + 'prefix-' + service, function(err, buffer) {
+      if(err && !err.notFound) {
+        return next(err);
+      }
+
+      if(!buffer || err.notFound) {
+        return next();
+      }
+
+      // we already have the prefix, call the callback
+      return callback(null, buffer.readUInt16BE());
+    };
+  }
+
+  function getUnused(next) {
+    self.store.get(self.dbPrefix + 'nextUnused', function(err, buffer) {
+      if(err && !err.notFound) {
+        return next(err);
+      }
+
+      if(!buffer || err.notFound) {
+        return next(null, 1);
+      }
+
+      return next(null, buffer.readUInt16BE());
+    }
+  }
+
+  function putPrefix(prefix, next) {
+    self.store.put(self.dbPrefix + 'prefix-' + service, prefix, function(err) {
+      if(err) {
+        return next(err);
+      }
+
+      next(null, prefix);
+    }
+  }
+
+  function putUnused(prefix, next) {
+    self.store.put(self.dbPrefix + 'nextUnused', prefix + 1, function(err) {
+      if(err) {
+        return next(err);
+      }
+
+      return next(null, prefix);
+    });
+  }
+};
+
 module.exports = DB;

lib/services/timestamp.js

@@ -0,0 +1,124 @@
+'use strict';
+var BaseService = require('../../service');
+var inherits = require('util').inherits;
+
+function TimestampService(options) {
+  BaseService.call(this, options);
+
+}
+
+inherits(TimestampService, BaseService);
+
+TimestampService.dependencies = [
+  'db'
+];
+
+TimestampService.prototype.start = function(callback) {
+  var self = this;
+
+  this.store = this.node.services.db.store;
+
+  this.node.services.db.getPrefix(this.name, function(err, prefix) {
+    if(err) {
+      return callback(err);
+    }
+
+    self.prefix = prefix;
+
+    callback();
+  });
+};
+
+TimestampService.prototype.stop = function(callback) {
+  setImmediate(callback);
+};
+
+TimestampService.prototype.blockHandler = function(block, connectBlock, callback) {
+  var self = this;
+
+  var action = 'put';
+  if (!connectBlock) {
+    action = 'del';
+  }
+
+  var operations = [];
+
+  function getLastTimestamp(next) {
+    var timestamp;
+
+    if(!block.header.prevHash) {
+      // Genesis block
+      return next(null, 0);
+    }
+
+    self.getTimestamp(block.header.prevHash.reverse().toString('hex'), next);
+  }
+
+  getLastTimestamp(function(err, lastTimestamp) {
+    if(err) {
+      return callback(err);
+    }
+
+    var timestamp = block.header.timestamp;
+    if(timestamp <= lastTimestamp) {
+      timestamp = lastTimestamp + 1;
+    }
+
+    operations.push({
+      type: action,
+      key: self._encodeTimestampBlockKey(timestamp),
+      value: self._encodeTimestampBlockValue(block.header.hash)
+    });
+
+    callback(null, operations);
+  });
+};
+
+Timestamp.prototype.getTimestamp = function(hash, callback) {
+  var key = this._encodeBlockTimestampKey(hash);
+  this.store.get(key, function(err, buffer) {
+    if(err) {
+      return callback(err);
+    }
+
+    return callback(null, this._decodeBlockTimestampValue(buffer));
+  });
+};
+
+TimestampService.prototype._encodeBlockTimestampKey = function(hash) {
+  return Buffer.concat([self.prefix, new Buffer(hash, 'hex')]);
+};
+
+TimestampService.prototype._decodeBlockTimestampKey = function(buffer) {
+  return buffer.slice(1).toString('hex');
+};
+
+TimestampService.prototype._encodeBlockTimestampValue = function(timestamp) {
+  var timestampBuffer = new Buffer(new Array(8));
+  timestampBuffer.writeDoubleBE(timestamp);
+  return timestampBuffer;
+};
+
+TimestampService.prototype._decodeBlockTimestampValue = function(buffer) {
+  return buffer.readDoubleBE(0);
+};
+
+TimestampService.prototype._encodeTimestampBlockKey = function(timestamp) {
+  var timestampBuffer = new Buffer(new Array(8));
+  timestampBuffer.writeDoubleBE(timestamp);
+  return Buffer.concat([self.prefix, timestampBuffer]);
+};
+
+TimestampService.prototype._decodeTimestampBlockKey = function(buffer) {
+  return buffer.readDoubleBE(1);
+};
+
+TimestampService.prototype._encodeTimestampBlockValue = function(hash) {
+  return new Buffer(hash, 'hex');
+};
+
+TimestampService.prototype._decodeTimestampBlockValue = function(buffer) {
+  return buffer.toString('hex');
+};
+
+module.exports = TimestampService;

lib/services/transaction.js

@@ -0,0 +1,75 @@
+'use strict';
+var BaseService = require('../../service');
+var inherits = require('util').inherits;
+
+function TransactionService(options) {
+  BaseService.call(this, options);
+
+}
+
+inherits(TransactionService, BaseService);
+
+TransactionService.dependencies = [
+  'db'
+];
+
+TransactionService.prototype.start = function(callback) {
+  var self = this;
+
+  this.store = this.node.services.db.store;
+
+  this.node.services.db.getPrefix(this.name, function(err, prefix) {
+    if(err) {
+      return callback(err);
+    }
+
+    self.prefix = prefix;
+
+    callback();
+  });
+};
+
+TransactionService.prototype.stop = function(callback) {
+  setImmediate(callback);
+};
+
+TransactionService.prototype.blockHandler = function(block, connectBlock, callback) {
+  var action = 'put';
+  if (!connectBlock) {
+    action = 'del';
+  }
+
+  var operations = [];
+
+  for(var i = 0; i < block.transactions.length; i++) {
+    var tx = block.transactions[i];
+
+    operations.push({
+      type: action,
+      key: this._encodeTransactionKey(tx.id),
+      value: this._encodeTransactionValue(tx)
+    });
+  }
+
+  setImmediate(function() {
+    callback(null, operations);
+  });
+};
+
+TransactionService.prototype._encodeTransactionKey = function(txid) {
+  return Buffer.concat([this.prefix, new Buffer(txid, 'hex')]);
+};
+
+TransactionService.prototype._decodeTransactionKey = function(buffer) {
+  return buffer.slice(1).toString('hex');
+};
+
+TransactionService.prototype._encodeTransactionValue = function(transaction) {
+  return transaction.uncheckedSerialize();
+};
+
+TransactionService.prototype._decodeTransactionValue = function(buffer) {
+  return new bitcore.Transaction(buffer);
+};
+
+module.exports = TransactionService;