options. misc.

lib/bcoin/blockdb.js

@@ -37,9 +37,12 @@ function BlockDB(options) {
 
   this.parser = new bcoin.protocol.parser();
 
-  this.data = new BlockData();
+  this.data = new BlockData(options);
-  this.unspentCache = new bcoin.lru(32 * 1024 * 1024);
+
-  this.txCache = new bcoin.lru(32 * 1024 * 1024);
+  this.cache = {
+    unspent: new bcoin.lru(32 * 1024 * 1024),
+    tx: new bcoin.lru(32 * 1024 * 1024)
+  };
 
   this.index = new levelup(this.file, {
     keyEncoding: 'ascii',
@@ -49,9 +52,12 @@ function BlockDB(options) {
     compression: true,
     cacheSize: 16 * 1024 * 1024,
     writeBufferSize: 8 * 1024 * 1024,
+    // https://leveldb.googlecode.com/git-history/master/doc/index.html
+    // Higher block size = better for iterators
+    // Lower block size = better for gets
     // blockSize: 4 * 1024,
-    // blockRestartInterval: 16,
     maxOpenFiles: 8192,
+    // blockRestartInterval: 16,
     db: bcoin.isBrowser
       ? require('memdown')
       : require('level' + 'down')
@@ -60,6 +66,56 @@ function BlockDB(options) {
 
 inherits(BlockDB, EventEmitter);
 
+BlockDB.prototype.migrate = function migrate(blockSize, compression, callback) {
+  var options, db, pending, stream, total, done;
+
+  options = utils.merge({}, this.index.options);
+
+  if (blockSize != null)
+    options.blockSize = blockSize;
+
+  if (compression != null)
+    options.compression = compression;
+
+  utils.print('Migrating DB with options:');
+  utils.print(options);
+
+  db = levelup(this.file + '.migrated', options);
+
+  stream = this.index.createReadStream();
+
+  pending = 0;
+  total = 0;
+
+  function onPut(err) {
+    if (err)
+      return callback(err);
+
+    if (++total % 10000 === 0)
+      utils.print('%d written.', total);
+
+    pending--;
+
+    if (done && !pending)
+      callback();
+  }
+
+  stream.on('data', function(data) {
+    pending++;
+    db.put(data.key, data.value, onPut);
+  });
+
+  stream.on('error', function(err) {
+    callback(err);
+  });
+
+  stream.on('end', function() {
+    done = true;
+    if (!pending)
+      callback();
+  });
+};
+
 BlockDB.prototype.createOffset = function createOffset(size, offset, height) {
   var buf = new Buffer(16);
   utils.writeU32(buf, size, 0);
@@ -138,6 +194,9 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) {
         }
 
         batch.del('u/t/' + input.prevout.hash + '/' + input.prevout.index);
+
+        if (self.options.cache)
+          self.cache.unspent.remove(input.prevout.hash + '/' + input.prevout.index);
       });
 
       tx.outputs.forEach(function(output) {
@@ -213,7 +272,7 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) {
 
       batch.del('t/t/' + hash);
 
-      self.fillTX2(tx, function(err) {
+      self.fillTX(tx, function(err) {
         if (err)
           return callback(err);
 
@@ -289,7 +348,47 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) {
   });
 };
 
-BlockDB.prototype.fillTX = function fillTX(tx, callback) {
+BlockDB.prototype.fillCoins = function fillCoins(txs, callback) {
+  var self = this;
+  var pending = txs.length;
+
+  callback = utils.asyncify(callback);
+
+  if (!pending)
+    return callback();
+
+  txs.forEach(function(tx) {
+    self.fillCoin(tx, function(err) {
+      if (err)
+        return callback(err);
+
+      if (!--pending)
+        callback();
+    });
+  });
+};
+
+BlockDB.prototype.fillTXs = function fillTXs(txs, callback) {
+  var self = this;
+  var pending = txs.length;
+
+  callback = utils.asyncify(callback);
+
+  if (!pending)
+    return callback();
+
+  txs.forEach(function(tx) {
+    self.fillTX(tx, function(err) {
+      if (err)
+        return callback(err);
+
+      if (!--pending)
+        callback();
+    });
+  });
+};
+
+BlockDB.prototype.fillCoin = function fillCoin(tx, callback) {
   var self = this;
   var pending = tx.inputs.length;
 
@@ -317,7 +416,7 @@ BlockDB.prototype.fillTX = function fillTX(tx, callback) {
   });
 };
 
-BlockDB.prototype.fillTX2 = function fillTX2(tx, callback) {
+BlockDB.prototype.fillTX = function fillTX(tx, callback) {
   var self = this;
   var pending = tx.inputs.length;
 
@@ -416,9 +515,11 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call
     var index = +parts[1];
     var id = hash + '/' + index;
     var record = self.parseOffset(data.value);
+
     pending++;
-    if (self.unspentCache.has(id)) {
+
-      coins.push(self.unspentCache.get(id));
+    if (self.options.cache && self.cache.unspent.has(id)) {
+      coins.push(self.cache.unspent.get(id));
       pending--;
       if (done) {
         if (!pending)
@@ -426,6 +527,7 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call
       }
       return;
     }
+
     self.data.getAsync(record.size, record.offset, function(err, data) {
       var coin;
 
@@ -438,6 +540,7 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call
         } catch (e) {
           return callback(e);
         }
+
         coin = bcoin.coin({
           version: 1,
           hash: hash,
@@ -447,7 +550,10 @@ BlockDB.prototype._getCoinsByAddress = function _getCoinsByAddress(address, call
           value: data.value,
           spent: false
         });
-        self.unspentCache.set(id, coin);
+
+        if (self.options.cache)
+          self.cache.unspent.set(id, coin);
+
         coins.push(coin);
       }
 
@@ -564,8 +670,8 @@ BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback)
 
     pending++;
 
-    if (self.txCache.has(hash)) {
+    if (self.options.cache && self.cache.tx.has(hash)) {
-      coins.push(self.txCache.get(hash));
+      coins.push(self.cache.tx.get(hash));
       pending--;
       if (done) {
         if (!pending)
@@ -588,11 +694,17 @@ BlockDB.prototype._getTXByAddress = function _getTXByAddress(address, callback)
         } catch (e) {
           return callback(e);
         }
+
         tx.height = record.height;
         tx.ts = entry.ts;
         tx.block = entry.hash;
-        self.txCache.set(hash, tx);
         txs.push(tx);
+
+        if (self.options.cache)
+          self.cache.tx.set(hash, tx);
+
+        if (self.options.paranoid && tx.hash('hex') !== hash)
+          return callback(new Error('BlockDB is corrupt. All is lost.'));
       }
 
       pending--;
@@ -645,6 +757,8 @@ BlockDB.prototype.getTX = function getTX(hash, callback) {
         tx.height = record.height;
         tx.ts = entry.ts;
         tx.block = entry.hash;
+        if (self.options.paranoid && tx.hash('hex') !== hash)
+          return callback(new Error('BlockDB is corrupt. All is lost.'));
       }
 
       return callback(null, tx);
@@ -682,6 +796,16 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) {
         }
         block._fileOffset = record.offset;
         block.height = record.height;
+        if (self.options.paranoid) {
+          if (typeof hash === 'number') {
+            hash = bcoin.chain.global.get(hash);
+            if (!hash)
+              return callback(null, block);
+            hash = hash.hash;
+          }
+          if (block.hash('hex') !== hash)
+            return callback(new Error('BlockDB is corrupt. All is lost.'));
+        }
       }
 
       return callback(null, block);
@@ -689,6 +813,15 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) {
   });
 };
 
+BlockDB.prototype.isSpent = function isSpent(hash, index, callback) {
+  this.getCoin(hash, index, function(err, coin) {
+    if (err)
+      return callback(err);
+
+    return callback(null, coin ? false : true);
+  });
+};
+
 /**
  * BlockData
  */

lib/bcoin/lru.js

@@ -12,13 +12,15 @@ var assert = utils.assert;
  * LRU
  */
 
-function LRU(maxSize) {
+function LRU(maxSize, getSize) {
   if (!(this instanceof LRU))
-    return new LRU(maxSize);
+    return new LRU(maxSize, getSize);
 
   this.data = {};
   this.size = 0;
   this.maxSize = maxSize;
+  this.getSize = getSize;
+
   this.head = null;
   this.tail = null;
 }
@@ -26,6 +28,9 @@ function LRU(maxSize) {
 LRU.prototype._getSize = function _getSize(item) {
   var keySize = item.key.length * 2;
 
+  if (this.getSize)
+    return keySize + this.getSize(item.value);
+
   if (item.value == null)
     return keySize + 1;
 
@@ -129,6 +134,7 @@ LRU.prototype._appendList = function appendList(item) {
 LRU.prototype._insertList = function insertList(ref, item) {
   assert(!item.next);
   assert(!item.prev);
+
   if (ref == null) {
     if (!this.head) {
       this.head = item;
@@ -140,9 +146,11 @@ LRU.prototype._insertList = function insertList(ref, item) {
     }
     return;
   }
+
   item.next = ref.next;
   item.prev = ref;
   ref.next = item;
+
   if (ref === this.tail)
     this.tail = item;
 };
@@ -160,6 +168,12 @@ LRU.prototype._removeList = function removeList(item) {
   if (item == this.tail)
     this.tail = item.prev || this.head;
 
+  if (!this.head)
+    assert(!this.tail);
+
+  if (!this.tail)
+    assert(!this.head);
+
   item.prev = null;
   item.next = null;
 };
@@ -167,6 +181,7 @@ LRU.prototype._removeList = function removeList(item) {
 LRU.prototype._keys = function _keys() {
   var keys = [];
   var item;
+
   for (item = this.head; item; item = item.next) {
     if (item === this.head)
       assert(!item.prev);
@@ -176,6 +191,7 @@ LRU.prototype._keys = function _keys() {
       assert(item === this.tail);
     keys.push(item.key);
   }
+
   return keys;
 };
 

lib/bcoin/mempool.js

@@ -141,6 +141,29 @@ Mempool.prototype.getTXByAddress = function getTXByAddress(addresses) {
   return txs;
 };
 
+Mempool.prototype.fillCoin =
+Mempool.prototype.fillTX = function fillTX(tx) {
+  var i, input, total;
+
+  total = 0;
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+
+    if (input.output) {
+      total++;
+      continue;
+    }
+
+    if (this.hasTX(input.prevout.hash)) {
+      input.output = this.getCoin(input.prevout.hash, input.prevout.index);
+      total++;
+    }
+  }
+
+  return total === tx.inputs.length;
+};
+
 Mempool.prototype.getAll = function getAll(hash) {
   return Object.keys(this.txs).map(function(key) {
     return this.txs[key];
@@ -175,7 +198,7 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback) {
 
   this._lockTX(tx);
 
-  this.storage.fillTX(tx, function(err) {
+  this.storage.fillCoin(tx, function(err) {
     var i, input, dup, height, ts, priority;
 
     self._unlockTX(tx);

lib/bcoin/node.js

@@ -35,7 +35,7 @@ function Node(options) {
   if (this.options.network)
     network.set(this.options.network);
 
-  this.storage = null;
+  this.block = null;
   this.mempool = null;
   this.pool = null;
   this.chain = null;
@@ -55,13 +55,13 @@ Node.prototype._init = function _init() {
 
   this.options.pool.type = 'full';
 
-  this.storage = new bcoin.blockdb(this.options.storage);
+  this.block = new bcoin.blockdb(this.options.block);
   this.mempool = new bcoin.mempool(this, this.options.mempool);
   this.pool = new bcoin.pool(this.options.pool);
   this.chain = this.pool.chain;
 
   this.pool.on('block', function(block, peer) {
-    self.storage.saveBlock(block, function(err) {
+    self.block.saveBlock(block, function(err) {
       if (err)
         throw err;
 
@@ -69,12 +69,12 @@ Node.prototype._init = function _init() {
       if (0)
       var hash = block.txs[0].hash('hex');
       if (0)
-      self.storage.getTX(hash, function(err, tx) {
+      self.block.getTX(hash, function(err, tx) {
         if (err) throw err;
         utils.print(tx);
       });
       if (0)
-      self.storage.getCoin(hash, 0, function(err, tx) {
+      self.block.getCoin(hash, 0, function(err, tx) {
         if (err) throw err;
         utils.print(tx);
       });
@@ -95,7 +95,7 @@ Node.prototype._init = function _init() {
 
   this.pool.on('fork', function(a, b) {
     [a, b].forEach(function(hash) {
-      self.storage.removeBlock(hash, function(err, block) {
+      self.block.removeBlock(hash, function(err, block) {
         if (err)
           throw err;
 
@@ -128,7 +128,7 @@ Node.prototype.getCoin = function getCoin(hash, index, callback) {
   if (this.mempool.isSpent(hash, index))
     return callback(null, null);
 
-  this.storage.getCoin(hash, index, function(err, coin) {
+  this.block.getCoin(hash, index, function(err, coin) {
     if (err)
       return callback(err);
 
@@ -147,7 +147,7 @@ Node.prototype.getCoinByAddress = function getCoinsByAddress(addresses, callback
 
   mempool = this.mempool.getCoinsByAddress(addresses);
 
-  this.storage.getCoinsByAddress(addresses, function(err, coins) {
+  this.block.getCoinsByAddress(addresses, function(err, coins) {
     if (err)
       return callback(err);
 
@@ -169,7 +169,7 @@ Node.prototype.getTX = function getTX(hash, callback) {
   if (tx)
     return callback(null, tx);
 
-  this.storage.getTX(hash, function(err, tx) {
+  this.block.getTX(hash, function(err, tx) {
     if (err)
       return callback(err);
 
@@ -185,12 +185,7 @@ Node.prototype.isSpent = function isSpent(hash, index, callback) {
   if (this.mempool.isSpent(hash, index))
     return callback(null, true);
 
-  this.storage.getCoin(hash, index, function(err, coin) {
+  this.block.isSpent(hash, index, callback);
-    if (err)
-      return callback(err);
-
-    return callback(null, coin ? false : true);
-  });
 };
 
 Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) {
@@ -201,7 +196,7 @@ Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) {
 
   mempool = this.mempool.getTXByAddress(addresses);
 
-  this.storage.getTXByAddress(addresses, function(err, txs) {
+  this.block.getTXByAddress(addresses, function(err, txs) {
     if (err)
       return callback(err);
 
@@ -209,8 +204,18 @@ Node.prototype.getTXByAddress = function getTXByAddress(addresses, callback) {
   });
 };
 
+Node.prototype.fillCoin = function fillCoin(tx, callback) {
+  callback = utils.asyncify(callback);
+  if (this.mempool.fillCoin(tx))
+    return callback();
+  this.block.fillCoin(tx, callback);
+};
+
 Node.prototype.fillTX = function fillTX(tx, callback) {
-  this.storage.fillTX(tx, callback);
+  callback = utils.asyncify(callback);
+  if (this.mempool.fillTX(tx))
+    return callback();
+  this.block.fillTX(tx, callback);
 };
 
 /**

lib/bcoin/pool.js

@@ -582,6 +582,14 @@ Pool.prototype._handleInv = function _handleInv(hashes, peer) {
   }
 };
 
+Pool.prototype._prehandleTX = function _prehandleTX(tx, peer, callback) {
+  return callback(null, tx);
+};
+
+Pool.prototype._prehandleBlock = function _prehandleBlock(block, peer, callback) {
+  return callback(null, block);
+};
+
 Pool.prototype._handleBlock = function _handleBlock(block, peer) {
   var self = this;
   var requested;