better pruning. data store improvements.

lib/bcoin/chaindb.js

@@ -1216,8 +1216,15 @@ ChainDB.prototype.hasBlock = function hasBlock(hash, callback) {
   var self = this;
   var id = 'b/b/' + hash;
 
-  if (typeof hash === 'number')
+  if (typeof hash === 'number') {
-    id = 'b/h/' + pad32(hash);
+    return this.getHash(hash, function(err, hash) {
+      if (err)
+        return callback(err);
+      if (!hash)
+        return callback();
+      return self.hasBlock(hash, callback);
+    });
+  }
 
   this.db.get(id, function(err, data) {
     if (err && err.type !== 'NotFoundError')
@@ -1330,6 +1337,7 @@ ChainDB.prototype._pruneBlock = function _pruneBlock(block, callback) {
 
   // For much more aggressive pruning, we could delete
   // the block headers 288 blocks before this one here as well.
+  batch.put('b/q/' + pad32(block.height + self.keepBlocks), block.hash());
 
   return utils.forEachSerial(block.txs, function(tx, next) {
     self._pruneTX(tx, batch, next);
@@ -1383,6 +1391,26 @@ ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) {
   }, callback);
 };
 
+ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) {
+  var self = this;
+
+  batch.put(
+    't/q/' + pad32(tx.height + self.keepBlocks) + '/' + tx.hash('hex'),
+    DUMMY);
+  return callback();
+
+  if (tx.isCoinbase())
+    return callback();
+
+  tx.inputs.forEach(function(input) {
+    batch.put(
+      't/q/' + pad32(tx.height + self.keepBlocks) + '/' + input.prevout.hash,
+      DUMMY);
+  });
+
+  callback();
+};
+
 ChainDB.prototype._removeTX = function _removeTX(hash, batch, callback) {
   var self = this;
   var uniq = {};
@@ -1468,13 +1496,23 @@ ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) {
     if (err)
       return callback(err);
 
-    if (hashes.length)
+    // if (hashes.length)
-      utils.debug('Retroactively pruning txs at height %d.', block.height);
+    //   utils.debug('Retroactively pruning txs at height %d.', block.height);
 
-    utils.forEachSerial(hashes, function(hash, next) {
+    self.db.get('b/q/' + pad32(block.height), function(err, hash) {
-      batch.del('t/q/' + pad32(block.height) + '/' + hash);
+      if (err && err.type !== 'NotFoundError')
-      self._removeTX(hash, batch, next);
+        return callback(err);
-    }, callback);
+
+      if (hash) {
+        batch.del('b/q/' + pad32(block.height));
+        batch.del('b/b/' + utils.toHex(hash));
+      }
+
+      utils.forEachSerial(hashes, function(hash, next) {
+        batch.del('t/q/' + pad32(block.height) + '/' + hash);
+        self._removeTX(hash, batch, next);
+      }, callback);
+    });
   }
 };
 

lib/bcoin/datastore.js

@@ -14,8 +14,10 @@ var assert = utils.assert;
 var fs = bcoin.fs;
 var pad32 = utils.pad32;
 
+var MAX_FILE_SIZE = 512 * 1024 * 1024;
 var MAX_FILE_SIZE = 1 * 1024 * 1024;
-var nullBuffer = new Buffer([0xff, 0xff, 0xff, 0xff]);
+var MAX_FILE_SIZE = 100 * 1024;
+var NULL_CHUNK = new Buffer([0xff, 0xff, 0xff, 0xff]);
 
 /**
  * DataStore
@@ -38,18 +40,21 @@ function DataStore(db, options) {
   if (!this.dir)
     this.dir = bcoin.prefix + '/store-' + network.type + '.db';
 
-  // Keep a pool of FDs open for disk cache benefits
   this._db = db;
   this.db = this;
+
+  this.busy = false;
+  this.jobs = [];
+
+  this.fileIndex = -1;
+
+  // Keep a pool of FDs open for disk cache benefits
   this.pool = new bcoin.lru(50, 1, function(key, value) {
     fs.close(value.fd, function(err) {
       if (err)
         self.emit('error', err);
     });
   });
-  this.busy = false;
-  this.jobs = [];
-  this.fileIndex = -1;
 
   this._init();
 }
@@ -206,7 +211,11 @@ DataStore.prototype.getData = function get(off, callback) {
   this.openFile(off.fileIndex, function(err, fd, fsize) {
     if (err)
       return callback(err);
-    return self.read(fd, off.offset, off.size, callback);
+    return self.read(fd, off.offset, off.size, function(err, data) {
+      if (err)
+        return callback(err);
+      return callback(null, data);
+    });
   });
 };
 
@@ -215,28 +224,35 @@ DataStore.prototype.get = function get(key, callback) {
   return this._db.get(key, function(err, offset) {
     if (err)
       return callback(err);
-    if (isDirect(offset))
+    if (isDirect(key, offset))
       return callback(null, offset);
     return self.getData(offset, callback);
   });
 };
 
 DataStore.prototype.batch = function batch(ops, options, callback) {
-  var batch = new Batch(this);
+  var batch;
+
+  if (!callback) {
+    callback = options;
+    options = null;
+  }
+
+  if (!options)
+    options = {};
+
+  batch = new Batch(this, options);
+
   if (ops) {
-    if (!callback) {
-      callback = options;
-      options = null;
-    }
-    if (!options)
-      options = {};
     batch.ops = ops;
     return batch.write(callback);
   }
+
   return batch;
 };
 
-function Batch(store) {
+function Batch(store, options) {
+  this.options = options;
   this.ops = [];
   this.store = store;
   this._db = store._db;
@@ -257,11 +273,16 @@ Batch.prototype.write = function write(callback) {
   if (!this._db)
     return callback(new Error('Already written.'));
 
-  batch = this._db.batch();
+  batch = this.options.sync
+    ? utils.syncBatch(this._db)
+    : this._db.batch();
+
+  if (this.options.sync)
+    this._db.fsync = true;
 
   utils.forEachSerial(this.ops, function(op, next) {
     if (op.type === 'put') {
-      if (isDirect(op.value)) {
+      if (isDirect(op.key, op.value)) {
         batch.put(op.key, op.value);
         return next();
       }
@@ -280,7 +301,7 @@ Batch.prototype.write = function write(callback) {
         if (!offset)
           return next();
         batch.del(op.key);
-        if (isDirect(offset))
+        if (isDirect(op.key, offset))
           return next();
         self.store.delData(offset, next);
       });
@@ -318,8 +339,9 @@ Iterator.prototype.seek = function seek(key) {
 // Store coins, chain entries, dummies, lookup
 // hashes directly in the db (unless they're
 // the same length as offset).
-function isDirect(data) {
+function isDirect(key, data) {
-  return data.length !== 12 && data.length <= 116;
+  return !/^(b\/b\/|t\/t\/)/.test(key);
+  return data.length !== 12 && (data.length <= 87 || data.length === 116);
 }
 
 Iterator.prototype.next = function next(callback) {
@@ -328,7 +350,7 @@ Iterator.prototype.next = function next(callback) {
       return callback(err);
 
     if (value) {
-      if (isDirect(value))
+      if (isDirect(key, value))
         return callback(null, key, value);
       return self.getData(value, function(err, data) {
         if (err)
@@ -359,7 +381,7 @@ utils.wrap = function wrap(callback, unlock) {
 
 DataStore.prototype.putData = function putData(data, callback, force) {
   var self = this;
-  var offset, undo;
+  var offset;
 
   var unlock = this._lock(putData, [data, callback], force);
   if (!unlock)
@@ -412,7 +434,7 @@ DataStore.prototype.put = function put(key, value, callback) {
 
   callback = utils.wrap(callback, unlock);
 
-  if (isDirect(value))
+  if (isDirect(key, value))
     return self._db.put(key, value, callback);
 
   return this.putData(value, function(err, offset) {
@@ -425,7 +447,6 @@ DataStore.prototype.put = function put(key, value, callback) {
 
 DataStore.prototype.readUndo = function readUndo(index, offset, callback) {
   var self = this;
-  var undo;
 
   return this.openFile(index, function(err, fd, fsize) {
     if (err)
@@ -440,26 +461,9 @@ DataStore.prototype.readUndo = function readUndo(index, offset, callback) {
   });
 };
 
-DataStore.prototype.isNull = function isNull(index, offset, size, callback) {
-  var self = this;
-  var undo;
-
-  return this.openFile(index, function(err, fd, fsize) {
-    if (err)
-      return callback(err);
-
-    return self.read(fd, offset, 4, function(err, data) {
-      if (err)
-        return callback(err);
-
-      return callback(null, utils.readU32(data, 0) === 0xffffffff);
-    });
-  });
-};
-
 DataStore.prototype.delData = function delData(off, callback, force) {
   var self = this;
-  var undo, index, offset, size;
+  var index, offset, size;
 
   var unlock = this._lock(delData, [off, callback], force);
   if (!unlock)
@@ -476,45 +480,44 @@ DataStore.prototype.delData = function delData(off, callback, force) {
     if (err)
       return callback(err);
 
-    return self.write(fd, offset, nullBuffer, function(err, data) {
+    // Overwrite the "fileIndex" in the undo chunk
+    return self.write(fd, offset + size, NULL_CHUNK, function(err) {
       if (err)
         return callback(err);
 
-      if (offset + size !== fsize)
+      if (offset + size + 12 !== fsize)
         return callback();
 
+      utils.debug('Pruning block files...');
+      utils.debug({ offset: offset, size: size, total: offset + size, fsize: fsize });
+
       // If we're deleting the last record, traverse
       // through the reverse linked list of undo offsets
       // until we hit a record that isn't deleted.
       // Truncate to the last deleted record's offset.
       (function next() {
+        if (offset === 0)
+          return done();
         self.readUndo(index, offset, function(err, undo) {
           if (err)
             return callback(err);
-          self.isNull(index, undo.offset, function(err, res) {
+          if (undo.fileIndex !== 0xffffffff)
-            if (!res)
+            return done();
-              return done();
+          offset = undo.offset;
-            offset = undo.offset;
+          if (offset === 0)
-            if (offset === 0)
+            return done();
-              return done();
+          return next();
-            return next();
-          });
         });
       })();
 
       function done() {
         // Delete the file if nothing is in it.
+        utils.debug('Truncating to %d', offset);
         if (offset === 0) {
-          return fs.close(fd, function(err) {
+          self.pool.remove(index);
-            if (err)
+          return fs.unlink(self.dir + '/f' + pad32(index), callback);
-              return callback(err);
-            self.pool.remove(index);
-            fs.unlink(self.dir + '/f' + pad32(index), callback);
-          });
         }
-        if (self.pool.has(index))
+        self.truncate(index, offset + 12, callback);
-          self.pool.get(index).size = offset;
-        self.truncate(index, offset, callback);
       }
     });
   });
@@ -533,8 +536,8 @@ DataStore.prototype.del = function del(key, callback, force) {
     if (err && err.type !== 'NotFoundError')
       return callback(err);
     if (!off)
-      return next();
+      return callback();
-    if (isDirect(off))
+    if (isDirect(key, off))
       return self._db.del(key, callback);
     self.delData(off, function(err) {
       if (err)
@@ -565,7 +568,7 @@ DataStore.prototype.truncate = function truncate(index, size, callback) {
 
   callback = utils.ensure(callback);
 
-  self.openFile(index, function(err, fd, fsize) {
+  this.openFile(index, function(err, fd, fsize) {
     if (err)
       return callback(err);