refactor: data management fixes.

lib/chain/chain.js

@@ -903,7 +903,7 @@ Chain.prototype.setBestChain = co(function* setBestChain(entry, block, prev) {
   }
 
   // Save block and connect inputs.
-  yield this.db.save(entry, block, view, true);
+  yield this.db.save(entry, block, view);
 
   this.tip = entry;
   this.height = entry.height;
@@ -1186,7 +1186,7 @@ Chain.prototype._add = co(function* add(block) {
     // our tip's. Add the block but do _not_
     // connect the inputs.
     if (entry.chainwork.cmp(this.tip.chainwork) <= 0) {
-      yield this.db.save(entry, block, null, false);
+      yield this.db.save(entry, block);
 
       this.emit('competitor', block, entry);
 

lib/chain/chaindb.js

@@ -16,6 +16,8 @@ var DUMMY = new Buffer([0]);
 var BufferWriter = require('../utils/writer');
 var BufferReader = require('../utils/reader');
 var spawn = require('../utils/spawn');
+var CoinView = require('./coinview');
+var Coins = require('./coins');
 var co = spawn.co;
 
 /*
@@ -232,7 +234,7 @@ ChainDB.prototype._open = co(function* open() {
 
     genesis = bcoin.chainentry.fromBlock(this.chain, block);
 
-    yield this.save(genesis, block, null, true);
+    yield this.save(genesis, block, new CoinView());
   }
 
   this.logger.info('Chain successfully loaded.');
@@ -544,28 +546,26 @@ ChainDB.prototype.get = co(function* get(hash) {
  * instead performed in {@link Chain#add}.
  * @param {ChainEntry} entry
  * @param {Block} block
- * @param {CoinView} view
+ * @param {CoinView?} view - Will not connect if null.
- * @param {Boolean} connect - Whether to connect the
- * block's inputs and add it as a tip.
  * @returns {Promise}
  */
 
-ChainDB.prototype.save = co(function* save(entry, block, view, connect) {
+ChainDB.prototype.save = co(function* save(entry, block, view) {
   var hash = block.hash();
   var height = new Buffer(4);
 
-  this.start();
-
   height.writeUInt32LE(entry.height, 0, true);
 
+  this.start();
+
   this.put(layout.h(hash), height);
   this.put(layout.e(hash), entry.toRaw());
 
   this.cacheHash.set(entry.hash, entry);
 
-  if (!connect) {
+  if (!view) {
     try {
-      yield this.saveBlock(block, view, false);
+      yield this.saveBlock(block);
     } catch (e) {
       this.drop();
       throw e;
@@ -579,13 +579,14 @@ ChainDB.prototype.save = co(function* save(entry, block, view, connect) {
   this.put(layout.H(entry.height), hash);
 
   try {
-    yield this.saveBlock(block, view, true);
+    yield this.saveBlock(block, view);
   } catch (e) {
     this.drop();
     throw e;
   }
 
   this.put(layout.R, this.pending.commit(hash));
+
   yield this.commit();
 });
 
@@ -814,13 +815,13 @@ ChainDB.prototype.has = co(function* has(height) {
  * @returns {Promise} - Returns {@link Block}.
  */
 
-ChainDB.prototype.saveBlock = co(function* saveBlock(block, view, connect) {
+ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) {
   if (this.options.spv)
     return block;
 
   this.put(layout.b(block.hash()), block.toRaw());
 
-  if (!connect)
+  if (!view)
     return block;
 
   yield this.connectBlock(block, view);
@@ -1107,7 +1108,7 @@ ChainDB.prototype.getCoin = co(function* getCoin(hash, index) {
   var coins = this.coinCache.get(hash);
 
   if (coins)
-    return bcoin.coins.parseCoin(coins, hash, index);
+    return Coins.parseCoin(coins, hash, index);
 
   coins = yield this.db.get(layout.c(hash));
 
@@ -1116,7 +1117,7 @@ ChainDB.prototype.getCoin = co(function* getCoin(hash, index) {
 
   this.coinCache.set(hash, coins);
 
-  return bcoin.coins.parseCoin(coins, hash, index);
+  return Coins.parseCoin(coins, hash, index);
 });
 
 /**
@@ -1129,7 +1130,7 @@ ChainDB.prototype.getCoins = co(function* getCoins(hash) {
   var coins = this.coinCache.get(hash);
 
   if (coins)
-    return bcoin.coins.fromRaw(coins, hash);
+    return Coins.fromRaw(coins, hash);
 
   coins = yield this.db.get(layout.c(hash));
 
@@ -1138,7 +1139,7 @@ ChainDB.prototype.getCoins = co(function* getCoins(hash) {
 
   this.coinCache.set(hash, coins);
 
-  return bcoin.coins.fromRaw(coins, hash);
+  return Coins.fromRaw(coins, hash);
 });
 
 /**
@@ -1401,7 +1402,7 @@ ChainDB.prototype.getFullBlock = co(function* getFullBlock(hash) {
  */
 
 ChainDB.prototype.getCoinView = co(function* getCoinView(block, callback) {
-  var view = new bcoin.coinview();
+  var view = new CoinView();
   var prevout = block.getPrevout();
   var i, prev, coins;
 

lib/db/lowlevelup.js

@@ -232,7 +232,8 @@ LowlevelUp.prototype.iterator = function iterator(options) {
     fillCache: options.fillCache || false,
     keyAsBuffer: this.bufferKeys,
     valueAsBuffer: true,
-    reverse: options.reverse || false
+    reverse: options.reverse || false,
+    highWaterMark: options.highWaterMark || 16 * 1024
   };
 
   // Workaround for a leveldown
@@ -303,19 +304,24 @@ LowlevelUp.prototype.has = co(function* has(key) {
 LowlevelUp.prototype.iterate = co(function* iterate(options) {
   var items = [];
   var parse = options.parse;
-  var iter, result, data;
+  var iter, item, data;
 
   assert(typeof parse === 'function', 'Parse must be a function.');
 
   iter = this.iterator(options);
 
   for (;;) {
-    result = yield iter.next();
+    item = yield iter.next();
 
-    if (!result)
+    if (!item)
       break;
 
-    data = parse(result.key, result.value);
+    try {
+      data = parse(item.key, item.value);
+    } catch (e) {
+      yield iter.end();
+      throw e;
+    }
 
     if (data)
       items.push(data);
@@ -357,7 +363,7 @@ LowlevelUp.prototype.clone = co(function* clone(path) {
   var opt = { keys: true, values: true };
   var hwm = 256 << 20;
   var total = 0;
-  var tmp, batch, iter, result;
+  var tmp, batch, iter, item;
 
   assert(!this.loading);
   assert(!this.closing);
@@ -374,12 +380,12 @@ LowlevelUp.prototype.clone = co(function* clone(path) {
   iter = this.iterator(opt);
 
   for (;;) {
-    result = yield iter.next();
+    item = yield iter.next();
 
-    if (!result)
+    if (!item)
       break;
 
-    batch.put(result.key, result.value);
+    batch.put(item.key, item.value);
     total += value.length;
 
     if (total >= hwm) {
@@ -387,6 +393,7 @@ LowlevelUp.prototype.clone = co(function* clone(path) {
       try {
         yield batch.write();
       } catch (e) {
+        yield iter.end();
         yield tmp.close();
         throw e;
       }

lib/mempool/mempool.js

@@ -576,8 +576,7 @@ Mempool.prototype.addTX = co(function* addTX(tx) {
 Mempool.prototype._addTX = co(function* _addTX(tx) {
   var lockFlags = constants.flags.STANDARD_LOCKTIME_FLAGS;
   var hash = tx.hash('hex');
-  var ret, entry, missing;
+  var ret, entry, result, exists;
-  var result, exists;
 
   assert(!tx.mutable, 'Cannot add mutable TX to mempool.');
 
@@ -665,10 +664,8 @@ Mempool.prototype._addTX = co(function* _addTX(tx) {
 
   yield this.fillAllCoins(tx);
 
-  if (!tx.hasCoins()) {
+  if (!tx.hasCoins())
-    missing = this.storeOrphan(tx);
+    return this.storeOrphan(tx);
-    return missing;
-  }
 
   entry = MempoolEntry.fromTX(tx, this.chain.height);
 

lib/wallet/walletdb.js

@@ -239,7 +239,7 @@ WalletDB.prototype.backup = function backup(path) {
  */
 
 WalletDB.prototype.getDepth = co(function* getDepth() {
-  var result, iter, depth;
+  var iter, item, depth;
 
   // This may seem like a strange way to do
   // this, but updating a global state when
@@ -256,14 +256,14 @@ WalletDB.prototype.getDepth = co(function* getDepth() {
     reverse: true
   });
 
-  result = yield iter.next();
+  item = yield iter.next();
 
-  if (!result)
+  if (!item)
     return 1;
 
   yield iter.end();
 
-  depth = layout.ww(result.key);
+  depth = layout.ww(item.key);
 
   return depth + 1;
 });

migrate/chaindb0to1.js

@@ -1,4 +1,6 @@
 var bcoin = require('../');
+var spawn = bcoin.spawn;
+var co = spawn.co;
 var assert = require('assert');
 var file = process.argv[2];
 
@@ -22,91 +24,93 @@ function makeKey(data) {
   return key;
 }
 
-function updateState(callback) {
+var checkVersion = co(function* checkVersion() {
-  var hash, batch, ver, p;
+  var data, ver;
+
+  console.log('Checking version.');
+
+  data = yield db.get('V');
+
+  if (!data)
+    return;
+
+  ver = data.readUInt32LE(0, true);
+
+  if (ver !== 0)
+    throw Error('DB is version ' + ver + '.');
+});
+
+var updateState = co(function* updateState() {
+  var data, hash, batch, ver, p;
 
   console.log('Updating chain state.');
 
-  db.get('R', function(err, data) {
+  data = yield db.get('R');
-    if (err)
-      return callback(err);
 
-    if (!data || data.length < 32)
+  if (!data || data.length < 32)
-      return callback(new Error('No chain state.'));
+    throw new Error('No chain state.');
 
-    hash = data.slice(0, 32);
+  hash = data.slice(0, 32);
 
-    p = new bcoin.writer();
+  p = new bcoin.writer();
-    p.writeHash(hash);
+  p.writeHash(hash);
-    p.writeU64(0);
+  p.writeU64(0);
-    p.writeU64(0);
+  p.writeU64(0);
-    p.writeU64(0);
+  p.writeU64(0);
-    p = p.render();
+  p = p.render();
 
-    batch = db.batch();
+  batch = db.batch();
 
-    batch.put('R', p);
+  batch.put('R', p);
 
-    ver = new Buffer(4);
+  ver = new Buffer(4);
-    ver.writeUInt32LE(1, 0, true);
+  ver.writeUInt32LE(1, 0, true);
-    batch.put('V', ver);
+  batch.put('V', ver);
 
-    batch.write(function(err) {
+  yield batch.write();
-      if (err)
-        return callback(err);
-      console.log('Updated chain state.');
-      callback();
-    });
-  });
-}
 
-function updateEndian(callback) {
+  console.log('Updated chain state.');
-  var lo = new Buffer('4800000000', 'hex');
+});
-  var hi = new Buffer('48ffffffff', 'hex');
+
+var updateEndian = co(function* updateEndian() {
   var batch = db.batch();
   var total = 0;
+  var iter, item;
 
   console.log('Updating endianness.');
   console.log('Iterating...');
 
-  db.iterate({
+  iter = db.iterator({
-    gte: lo,
+    gte: new Buffer('4800000000', 'hex'),
-    lte: hi,
+    lte: new Buffer('48ffffffff', 'hex'),
-    values: true,
+    values: true
-    parse: function(key, value) {
-      batch.del(key);
-      batch.put(makeKey(key), value);
-      total++;
-    }
-  }, function(err) {
-    if (err)
-      throw err;
-
-    console.log('Migrating %d items.', total);
-
-    batch.write(function(err) {
-      if (err)
-        throw err;
-      console.log('Migrated endianness.');
-      callback();
-    });
   });
-}
 
-db.open(function(err) {
+  for (;;) {
-  if (err)
+    item = yield iter.next();
-    throw err;
 
-  console.log('Opened %s.', file);
+    if (!item)
+      break;
 
-  updateState(function(err) {
+    batch.del(item.key);
-    if (err)
+    batch.put(makeKey(item.key), item.value);
-      throw err;
+    total++;
-    updateEndian(function(err) {
+  }
-      if (err)
+
-        throw err;
+  console.log('Migrating %d items.', total);
-      console.log('Migration complete.');
+
-      process.exit(0);
+  yield batch.write();
-    });
+
-  });
+  console.log('Migrated endianness.');
+});
+
+spawn(function *() {
+  yield db.open();
+  console.log('Opened %s.', file);
+  yield checkVersion();
+  yield updateState();
+  yield updateEndian();
+}).then(function() {
+  console.log('Migration complete.');
+  process.exit(0);
 });