Address Service: Start to use streams for memory optimization with large queries

lib/services/address/constants.js

@@ -0,0 +1,41 @@
+'use strict';
+
+var exports = {};
+
+exports.PREFIXES = {
+  OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
+  SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
+  SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
+};
+
+exports.MEMPREFIXES = {
+  OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address
+  SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address
+  SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output
+};
+
+// To save space, we're only storing the PubKeyHash or ScriptHash in our index.
+// To avoid intentional unspendable collisions, which have been seen on the blockchain,
+// we must store the hash type (PK or Script) as well.
+exports.HASH_TYPES = {
+  PUBKEY: new Buffer('01', 'hex'),
+  REDEEMSCRIPT: new Buffer('02', 'hex')
+};
+
+// Translates from our enum type back into the hash types returned by
+// bitcore-lib/address.
+exports.HASH_TYPES_READABLE = {
+  '01': 'pubkeyhash',
+  '02': 'scripthash'
+};
+
+exports.HASH_TYPES_MAP = {
+  'pubkeyhash': exports.HASH_TYPES.PUBKEY,
+  'scripthash': exports.HASH_TYPES.REDEEMSCRIPT
+};
+
+exports.SPACER_MIN = new Buffer('00', 'hex');
+exports.SPACER_MAX = new Buffer('ff', 'hex');
+
+module.exports = exports;
+

lib/services/address/encoding.js

@@ -0,0 +1,211 @@
+'use strict';
+
+var bitcore = require('bitcore-lib');
+var BufferReader = bitcore.encoding.BufferReader;
+var Address = bitcore.Address;
+var PublicKey = bitcore.PublicKey;
+var constants = require('./constants');
+var $ = bitcore.util.preconditions;
+
+var exports = {};
+
+exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) {
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  var key = Buffer.concat([
+    txidBuffer,
+    outputIndexBuffer
+  ]);
+  return key.toString('binary');
+};
+
+exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) {
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  var key = Buffer.concat([
+    constants.PREFIXES.OUTPUTS,
+    hashBuffer,
+    hashTypeBuffer,
+    constants.SPACER_MIN,
+    heightBuffer,
+    txidBuffer,
+    outputIndexBuffer
+  ]);
+  return key;
+};
+
+exports.decodeOutputKey = function(buffer) {
+  var reader = new BufferReader(buffer);
+  var prefix = reader.read(1);
+  var hashBuffer = reader.read(20);
+  var hashTypeBuffer = reader.read(1);
+  var spacer = reader.read(1);
+  var height = reader.readUInt32BE();
+  var txid = reader.read(32);
+  var outputIndex = reader.readUInt32BE();
+  return {
+    prefix: prefix,
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    height: height,
+    txid: txid,
+    outputIndex: outputIndex
+  };
+};
+
+exports.encodeOutputValue = function(satoshis, scriptBuffer) {
+  var satoshisBuffer = new Buffer(8);
+  satoshisBuffer.writeDoubleBE(satoshis);
+  return Buffer.concat([satoshisBuffer, scriptBuffer]);
+};
+
+exports.decodeOutputValue = function(buffer) {
+  var satoshis = buffer.readDoubleBE(0);
+  var scriptBuffer = buffer.slice(8, buffer.length);
+  return {
+    satoshis: satoshis,
+    scriptBuffer: scriptBuffer
+  };
+};
+
+exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) {
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  return Buffer.concat([
+    constants.PREFIXES.SPENTS,
+    hashBuffer,
+    hashTypeBuffer,
+    constants.SPACER_MIN,
+    heightBuffer,
+    prevTxIdBuffer,
+    outputIndexBuffer
+  ]);
+};
+
+exports.decodeInputKey = function(buffer) {
+  var reader = new BufferReader(buffer);
+  var prefix = reader.read(1);
+  var hashBuffer = reader.read(20);
+  var hashTypeBuffer = reader.read(1);
+  var spacer = reader.read(1);
+  var height = reader.readUInt32BE();
+  var prevTxId = reader.read(32);
+  var outputIndex = reader.readUInt32BE();
+  return {
+    prefix: prefix,
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    height: height,
+    prevTxId: prevTxId,
+    outputIndex: outputIndex
+  };
+};
+
+exports.encodeInputValue = function(txidBuffer, inputIndex) {
+  var inputIndexBuffer = new Buffer(4);
+  inputIndexBuffer.writeUInt32BE(inputIndex);
+  return Buffer.concat([
+    txidBuffer,
+    inputIndexBuffer
+  ]);
+};
+
+exports.decodeInputValue = function(buffer) {
+  var txid = buffer.slice(0, 32);
+  var inputIndex = buffer.readUInt32BE(32);
+  return {
+    txid: txid,
+    inputIndex: inputIndex
+  };
+};
+
+exports.encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) {
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  return Buffer.concat([
+    constants.PREFIXES.SPENTSMAP,
+    outputTxIdBuffer,
+    outputIndexBuffer
+  ]);
+};
+
+exports.decodeInputKeyMap = function(buffer) {
+  var txid = buffer.slice(1, 33);
+  var outputIndex = buffer.readUInt32BE(33);
+  return {
+    outputTxId: txid,
+    outputIndex: outputIndex
+  };
+};
+
+exports.encodeInputValueMap = function(inputTxIdBuffer, inputIndex) {
+  var inputIndexBuffer = new Buffer(4);
+  inputIndexBuffer.writeUInt32BE(inputIndex);
+  return Buffer.concat([
+    inputTxIdBuffer,
+    inputIndexBuffer
+  ]);
+};
+
+exports.decodeInputValueMap = function(buffer) {
+  var txid = buffer.slice(0, 32);
+  var inputIndex = buffer.readUInt32BE(32);
+  return {
+    inputTxId: txid,
+    inputIndex: inputIndex
+  };
+};
+
+exports.getAddressInfo = function(addressStr) {
+  var addrObj = bitcore.Address(addressStr);
+  var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type];
+  
+  return {
+    hashBuffer: addrObj.hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    hashTypeReadable: addrObj.type
+  };
+};
+
+/**
+ * This function is optimized to return address information about an output script
+ * without constructing a Bitcore Address instance.
+ * @param {Script} - An instance of a Bitcore Script
+ * @param {Network|String} - The network for the address
+ */
+exports.extractAddressInfoFromScript = function(script, network) {
+  $.checkArgument(network, 'Second argument is expected to be a network');
+  var hashBuffer;
+  var addressType;
+  var hashTypeBuffer;
+  if (script.isPublicKeyHashOut()) {
+    hashBuffer = script.chunks[2].buf;
+    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
+    addressType = Address.PayToPublicKeyHash;
+  } else if (script.isScriptHashOut()) {
+    hashBuffer = script.chunks[1].buf;
+    hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT;
+    addressType = Address.PayToScriptHash;
+  } else if (script.isPublicKeyOut()) {
+    var pubkey = script.chunks[0].buf;
+    var address = Address.fromPublicKey(new PublicKey(pubkey), network);
+    hashBuffer = address.hashBuffer;
+    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
+    // pay-to-publickey doesn't have an address, however for compatibility
+    // purposes, we can create an address
+    addressType = Address.PayToPublicKeyHash;
+  } else {
+    return false;
+  }
+  return {
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    addressType: addressType
+  };
+};
+
+module.exports = exports;

lib/services/address/streams/inputs-transform.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var Transform = require('stream').Transform;
+var inherits = require('util').inherits;
+var bitcore = require('bitcore-lib');
+var encodingUtil = require('../encoding');
+var $ = bitcore.util.preconditions;
+
+function InputsTransformStream(options) {
+  $.checkArgument(options.address instanceof bitcore.Address);
+  Transform.call(this, {
+    objectMode: true
+  });
+  this._address = options.address;
+  this._addressStr = this._address.toString();
+  this._tipHeight = options.tipHeight;
+}
+inherits(InputsTransformStream, Transform);
+
+InputsTransformStream.prototype._transform = function(chunk, encoding, callback) {
+  var self = this;
+
+  var key = encodingUtil.decodeInputKey(chunk.key);
+  var value = encodingUtil.decodeInputValue(chunk.value);
+
+  var input = {
+    address: this._addressStr,
+    hashType: this._address.type,
+    txid: value.txid.toString('hex'),
+    inputIndex: value.inputIndex,
+    height: key.height,
+    confirmations: this._tipHeight - key.height + 1
+  };
+
+  self.push(input);
+  callback();
+
+};
+
+module.exports = InputsTransformStream;

lib/services/address/streams/outputs-transform.js

@@ -0,0 +1,42 @@
+'use strict';
+
+var Transform = require('stream').Transform;
+var inherits = require('util').inherits;
+var bitcore = require('bitcore-lib');
+var encodingUtil = require('../encoding');
+var $ = bitcore.util.preconditions;
+
+function OutputsTransformStream(options) {
+  Transform.call(this, {
+    objectMode: true
+  });
+  $.checkArgument(options.address instanceof bitcore.Address);
+  this._address = options.address;
+  this._addressStr = this._address.toString();
+  this._tipHeight = options.tipHeight;
+}
+inherits(OutputsTransformStream, Transform);
+
+OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) {
+  var self = this;
+
+  var key = encodingUtil.decodeOutputKey(chunk.key);
+  var value = encodingUtil.decodeOutputValue(chunk.value);
+
+  var output = {
+    address: this._addressStr,
+    hashType: this._address.type,
+    txid: key.txid.toString('hex'), //TODO use a buffer
+    outputIndex: key.outputIndex,
+    height: key.height,
+    satoshis: value.satoshis,
+    script: value.scriptBuffer.toString('hex'), //TODO use a buffer
+    confirmations: this._tipHeight - key.height + 1
+  };
+
+  self.push(output);
+  callback();
+
+};
+
+module.exports = OutputsTransformStream;