misc improvements.

lib/bcoin.js

@@ -15,6 +15,7 @@ bcoin.prefix = process.env.BCOIN_PREFIX || process.env.HOME + '/.bcoin';
 bcoin.debug = +process.env.BCOIN_DEBUG === 1;
 bcoin.debugFile = +process.env.BCOIN_DEBUGFILE !== 0;
 bcoin.profile = +process.env.BCOIN_PROFILE === 1;
+bcoin.fresh = +process.env.BCOIN_FRESH === 1;
 
 bcoin.ensurePrefix = function ensurePrefix() {
   if (!bcoin.fs)
@@ -25,6 +26,9 @@ bcoin.ensurePrefix = function ensurePrefix() {
 
   bcoin._ensured = true;
 
+  if (bcoin.fresh && bcoin.prefix.indexOf('bcoin') !== -1)
+    bcoin.rimraf(bcoin.prefix);
+
   try {
     bcoin.fs.statSync(bcoin.prefix);
   } catch (e) {
@@ -32,6 +36,17 @@ bcoin.ensurePrefix = function ensurePrefix() {
   }
 };
 
+bcoin.rimraf = function rimraf(file) {
+  if (!bcoin.cp)
+    return;
+
+  assert(typeof file === 'string');
+  assert(file !== '/');
+  assert(file !== process.env.HOME);
+
+  bcoin.cp.execFileSync('rm', ['-rf', file], { stdio: 'ignore' });
+};
+
 bcoin.bn = require('bn.js');
 bcoin.elliptic = require('elliptic');
 

lib/bcoin/blockdb.js

@@ -44,13 +44,10 @@ function BlockDB(node, options) {
   this.node = node;
 
   this.cache = {
-    unspent: new bcoin.lru(32 * 1024 * 1024, function() { return 80; }),
+    unspent: new bcoin.lru(200000),
-    tx: new bcoin.lru(32 * 1024 * 1024, function(key, tx) { return tx.getSize(); })
+    tx: new bcoin.lru(50000)
   };
 
-  if (+process.env.BCOIN_FRESH === 1 && bcoin.cp)
-    bcoin.cp.execFileSync('rm', ['-rf', this.file], { stdio: 'ignore' });
-
   this.db = new levelup(this.file, {
     keyEncoding: 'ascii',
     valueEncoding: 'binary',
@@ -59,9 +56,7 @@ function BlockDB(node, options) {
     compression: true,
     cacheSize: 16 * 1024 * 1024,
     writeBufferSize: 8 * 1024 * 1024,
-    // blockSize: 4 * 1024,
     maxOpenFiles: 8192,
-    // blockRestartInterval: 16,
     db: bcoin.isBrowser
       ? require('level-js')
       : require('level' + 'down')
@@ -804,7 +799,12 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) {
         return callback(err);
       }
 
-      block = bcoin.block.fromCompact(data);
+      try {
+        block = bcoin.block.fromCompact(data);
+      } catch (e) {
+        return callback(e);
+      }
+
       block.txs = [];
 
       utils.forEach(block.hashes, function(hash, next, i) {

lib/bcoin/chain.js

@@ -163,14 +163,6 @@ Chain.prototype._init = function _init() {
             self.tip = tip;
             self.height = tip.height;
 
-            // Start fsyncing writes once we're no
-            // longer dealing with historical data.
-            if (tip.height >= network.checkpoints.lastHeight) {
-              self.db.fsync = true;
-              if (self.blockdb)
-                self.blockdb.fsync = true;
-            }
-
             self.loading = false;
             self.emit('load');
           });
@@ -991,7 +983,7 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) {
 
         // Start fsyncing writes once we're no
         // longer dealing with historical data.
-        if (entry.height >= network.checkpoints.lastHeight) {
+        if (self.isFull()) {
           self.db.fsync = true;
           if (self.blockdb)
             self.blockdb.fsync = true;
@@ -1393,7 +1385,7 @@ Chain.prototype.add = function add(initial, peer, callback, force) {
     self.total += total;
 
     // Take heap snapshot for debugging.
-    if (self.total % 10 === 0)
+    if (self.total % 20 === 0)
       bcoin.profiler.snapshot();
 
     utils.nextTick(function() {

lib/bcoin/chaindb.js

@@ -65,10 +65,8 @@ function ChainDB(node, chain, options) {
   else
     this._cacheWindow = network.block.majorityWindow + 1;
 
-  this.cacheHash = new NullCache(this._cacheWindow * 200); // (not hashcash)
+  this.cacheHash = new bcoin.lru(this._cacheWindow);
-  this.cacheHeight = new NullCache(this._cacheWindow * 200);
+  this.cacheHeight = new bcoin.lru(this._cacheWindow);
-  // this.cacheHash = new bcoin.lru(this._cacheWindow, function() { return 1; }); // (not hashcash)
-  // this.cacheHeight = new bcoin.lru(this._cacheWindow, function() { return 1; });
 
   this._init();
 }
@@ -80,9 +78,6 @@ ChainDB.prototype._init = function _init() {
 
   bcoin.ensurePrefix();
 
-  if (+process.env.BCOIN_FRESH === 1 && bcoin.cp)
-    bcoin.cp.execFileSync('rm', ['-rf', this.file], { stdio: 'ignore' });
-
   this.db = new levelup(this.file, {
     keyEncoding: 'ascii',
     valueEncoding: 'binary',
@@ -91,9 +86,7 @@ ChainDB.prototype._init = function _init() {
     compression: false,
     cacheSize: 16 * 1024 * 1024,
     writeBufferSize: 8 * 1024 * 1024,
-    // blockSize: 4 * 1024,
     maxOpenFiles: 8192,
-    // blockRestartInterval: 16,
     db: bcoin.isBrowser
       ? require('level-js')
       : require('level' + 'down')

lib/bcoin/lru.js

@@ -37,22 +37,13 @@ LRU.prototype._getSize = function _getSize(item) {
   if (typeof item.value === 'number')
     return keySize + 4;
 
-  if (item.value._raw)
-    return keySize + item.value._raw.length;
-
-  if (item.value.getSize)
-    return keySize + item.value.getSize();
-
-  if (typeof item.value._size === 'number')
-    return keySize + item.value._size;
-
   if (typeof item.value === 'string')
     return keySize + item.value.length * 2;
 
   if (typeof item.value.length === 'number')
     return keySize + item.value.length;
 
-  return keySize + 1;
+  return 1;
 };
 
 LRU.prototype._compact = function _compact() {
@@ -220,7 +211,7 @@ LRU.prototype._removeList = function removeList(item) {
   item.next = null;
 };
 
-LRU.prototype._keys = function _keys() {
+LRU.prototype.keys = function keys() {
   var keys = [];
   var item;
 
@@ -237,66 +228,15 @@ LRU.prototype._keys = function _keys() {
   return keys;
 };
 
-var a1 = '1';
+LRU.prototype.iterator = function iterator() {
-var a2 = '2';
+  return {
-var a3 = '3';
+    item: { next: this.head },
-var a4 = '4';
+    next: function() {
-var a5 = '5';
+      this.item = this.item.next;
-var a6 = '6';
+      return this.item;
-var lru = new LRU(4, function() { return 1; });
+    }
-lru.set('a1', a1);
+  };
-assert(lru.get('a1') === '1');
+};
-assert(lru.size === 1);
-assert(lru.head.key === 'a1' && lru.tail.key === 'a1' && !lru.head.prev && !lru.head.next);
-lru._keys();
-// console.log(lru._keys());
-lru.set('a2', a2);
-assert(lru.get('a2') === '2');
-assert(lru.size === 2);
-assert(lru.head.key === 'a1' && lru.tail.key === 'a2'
-  && !lru.head.prev && lru.head.next.key === 'a2'
-  && !lru.tail.next && lru.tail.prev.key === 'a1');
-lru._keys();
-// console.log(lru._keys());
-lru.set('a3', a3);
-assert(lru.get('a3') === '3');
-assert(lru.size === 3);
-lru._keys();
-// console.log(lru._keys());
-lru.set('a3', a3);
-assert(lru.get('a3') === '3');
-assert(lru.size === 3);
-lru._keys();
-// console.log(lru._keys());
-lru.set('a4', a4);
-assert(lru.get('a4') === '4');
-assert(lru.size === 4);
-lru._keys();
-// console.log(lru._keys());
-assert(lru.get('a1'));
-lru.remove('a1');
-lru._keys();
-// console.log(lru._keys());
-var _a3 = lru.head.next;
-assert(_a3.key === 'a3');
-assert(_a3.prev === lru.head && _a3.next === lru.tail);
-assert(lru.head.key === 'a2' && lru.tail.key === 'a4'
-  && !lru.head.prev && lru.head.next.key === 'a3'
-  && !lru.tail.next && lru.tail.prev.key === 'a3');
-lru.set('a5', a5);
-assert(lru.get('a5') === '5');
-assert(lru.size === 4);
-assert(!lru.get('a1'));
-lru._keys();
-// console.log(lru._keys());
-lru.get('a2');
-lru._keys();
-lru.set('a6', a6);
-assert(lru.get('a6') === '6');
-assert(lru.size === 4);
-assert(!lru.get('a3'));
-lru._keys();
-// console.log(lru._keys());
 
 /**
  * Expose

lib/bcoin/node.js

@@ -48,7 +48,7 @@ Fullnode.prototype._init = function _init() {
   // used for tx retrieval.
   this.blockdb = new bcoin.blockdb(this, {
     cache: false,
-    fsync: true
+    fsync: false
   });
 
   // Mempool needs access to blockdb.
@@ -59,7 +59,7 @@ Fullnode.prototype._init = function _init() {
   // Chain needs access to blockdb.
   this.chain = new bcoin.chain(this, {
     preload: false,
-    fsync: true
+    fsync: false
   });
 
   // Pool needs access to the chain.

lib/bcoin/profiler.js

@@ -161,6 +161,7 @@ exports.takeSnapshot = function takeSnapshot(name) {
 };
 
 exports.snapshot = function snapshot(name, callback) {
+  var mem = process.memoryUsage();
   var snapshot;
 
   if (typeof name === 'function') {
@@ -168,6 +169,9 @@ exports.snapshot = function snapshot(name, callback) {
     name = null;
   }
 
+  utils.debug('Memory: rss=%dmb, heap=%dmb',
+    utils.mb(mem.rss), utils.mb(mem.heapUsed));
+
   if (!profiler)
     return callback ? utils.nextTick(callback) : null;
 

lib/bcoin/utils.js

@@ -1741,20 +1741,25 @@ function SyncBatch(db) {
 }
 
 SyncBatch.prototype.put = function put(key, value) {
-  this.ops.push({ type: 'put', key: key, value: value, sync: true });
+  assert(this.ops, 'Batch was already written.');
+  this.ops.push({ type: 'put', key: key, value: value });
 };
 
 SyncBatch.prototype.del = function del(key) {
-  this.ops.push({ type: 'del', key: key, sync: true });
+  assert(this.ops, 'Batch was already written.');
+  this.ops.push({ type: 'del', key: key });
 };
 
 SyncBatch.prototype.write = function write(callback) {
+  assert(this.ops, 'Batch was already written.');
   this.db.batch(this.ops, { sync: true }, callback);
   this.ops.length = 0;
   delete this.ops;
+  delete this.db;
 };
 
 SyncBatch.prototype.clear = function clear() {
+  assert(this.ops, 'Batch was already written.');
   this.ops.length = 0;
 };
 

lib/bcoin/walletdb.js

@@ -104,11 +104,9 @@ WalletDB.prototype._init = function _init() {
       createIfMissing: true,
       errorIfExists: false,
       compression: true,
-      cacheSize: 1 * 1024 * 1024,
+      cacheSize: 4 * 1024 * 1024,
-      writeBufferSize: 1 * 1024 * 1024,
+      writeBufferSize: 4 * 1024 * 1024,
-      // blockSize: 4 * 1024,
       maxOpenFiles: 1024,
-      // blockRestartInterval: 16,
       db: bcoin.isBrowser
         ? require('level-js')
         : require('level' + 'down')
@@ -576,8 +574,8 @@ WalletDB.prototype.removeBlock = function removeBlock(block, callback) {
 
   callback = utils.ensure(callback);
 
-  utils.forEach(block.txs, function(tx, next) {
+  utils.forEachSerial(block.txs, function(tx, next) {
-    self.tx.unconfirm(tx.hash('hex'));
+    self.tx.unconfirm(tx.hash('hex'), next);
   }, callback);
 };