Merge pull request #81 from pnagurny/feature/blockchain-cache

Cache blockchain data for fast startup time

lib/blockchain.js

@@ -18,6 +18,7 @@ function BlockChain() {
   };
   this.next = {};
   this.prev = {};
+  this.cache = [];
 }
 
 BlockChain.NULL = NULL;
@@ -112,6 +113,7 @@ BlockChain.prototype.confirm = function(hash) {
   this.next[prevHash] = hash;
   this.hashByHeight[height] = hash;
   this.height[hash] = height;
+  this.cache.push(this.getCachedDataForHash(hash));
 };
 
 BlockChain.prototype.unconfirm = function(hash) {
@@ -123,6 +125,7 @@ BlockChain.prototype.unconfirm = function(hash) {
   delete this.next[prevHash];
   delete this.hashByHeight[height];
   delete this.height[hash];
+  this.cache.pop();
 };
 
 BlockChain.prototype.hasData = function(hash) {
@@ -178,4 +181,24 @@ BlockChain.prototype.getCurrentHeight = function() {
   return this.height[this.tip];
 };
 
+BlockChain.prototype.loadFromCache = function(cache) {
+  for(var i = 0; i < cache.length; i++) {
+    this.prev[cache[i].hash] = cache[i].prevHash;
+    this.work[cache[i].hash] = cache[i].work;
+    this.confirm(cache[i].hash);
+  }
+};
+
+BlockChain.prototype.getCache = function() {
+  return this.cache;
+};
+
+BlockChain.prototype.getCachedDataForHash = function(hash) {
+  return {
+    hash: hash,
+    prevHash: this.prev[hash],
+    work: this.work[hash]
+  };
+};
+
 module.exports = BlockChain;

lib/networkmonitor.js

@@ -19,6 +19,7 @@ function NetworkMonitor(eventBus, peer) {
   this.messages = new p2p.Messages({
     network: this.peer.network,
   });
+  this.ignoreInv = true;
   this.setupPeer(peer);
 }
 util.inherits(NetworkMonitor, EventEmitter);
@@ -45,6 +46,10 @@ NetworkMonitor.prototype.setupPeer = function(peer) {
     self.maxHeight = m.startHeight;
   });
   peer.on('inv', function(m) {
+    if(self.ignoreInv) {
+      return;
+    }
+
     self.emit('inv', m.inventory);
     // TODO only ask for data if tx or block is unknown
     peer.sendMessage(self.messages.GetData(m.inventory));

lib/node.js

@@ -37,6 +37,7 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService
   this.blockService = blockService;
 
   this.blockCache = {};
+  this.syncStrategy = 'rpc';
   this.initialize();
 };
 util.inherits(BitcoreNode, EventEmitter);
@@ -101,6 +102,8 @@ BitcoreNode.prototype.initialize = function() {
     var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex');
     self.blockCache[block.hash] = block;
     if (!self.blockchain.hasData(prevHash)) {
+      self.syncStrategy = 'p2p';
+      self.networkMonitor.ignoreInv = false;
       self._requestFromTip();
       return;
     }
@@ -125,6 +128,12 @@ BitcoreNode.prototype.initialize = function() {
           delete self.blockCache[deleteHash];
         }
       })
+      .then(function() {
+        // Update header cache every 100 blocks
+        if(block.height % 100 === 0) {
+          return self.blockService.saveBlockchainCache(self.blockchain.getCache());
+        }
+      })
       .catch(function(error) {
         self.stop(error);
       });
@@ -218,10 +227,45 @@ BitcoreNode.prototype.sync = function() {
   var self = this;
   this.networkMonitor.on('ready', function(reportedMaxHeight) {
     self.reportedMaxHeight = reportedMaxHeight;
-    self._requestFromTip();
+    if(self.reportedMaxHeight === self.getCurrentHeight()) {
+      self.syncStrategy = 'p2p';
+    }
+
+    if(self.syncStrategy === 'p2p') {
+      self.networkMonitor.ignoreInv = false;
+      self._requestFromTip();
+    } else {
+      self._rpcFetch();
+    }
   });
 };
 
+BitcoreNode.prototype._rpcFetch = function() {
+  var self = this;
+
+  self.blockService.getBlockByHeight(self.getCurrentHeight() + 1)
+    .then(function(block) {
+      self.bus.process(block)
+      .catch(function(err) {
+        self.stop(err);
+      });
+    })
+    .then(function() {
+      if(self.syncStrategy === 'rpc') {
+        if(self.reportedMaxHeight === self.getCurrentHeight()) {
+          // We are synced
+          self.syncStrategy = 'p2p';
+          self.networkMonitor.ignoreInv = false;
+          return;
+        }
+        self._rpcFetch();
+      }
+    })
+    .catch(function(error) {
+      self.stop(error);
+    });
+};
+
 var errors = require('./errors');
 BitcoreNode.errors = errors;
 module.exports = BitcoreNode;

lib/services/block.js

@@ -38,7 +38,8 @@ var Index = {
   height: 'bh-', // bh-<hash> -> height (-1 means disconnected)
   tip: 'tip', // tip -> { hash: hex, height: int }, the latest tip
   work: 'wk-', // wk-<hash> -> amount of work for block
-  header: 'header-' // header-<hash> -> JSON for block header
+  header: 'header-', // header-<hash> -> JSON for block header
+  blockchain: 'bc-'
 };
 _.extend(Index, {
   getNextBlock: helper(Index.next),
@@ -132,30 +133,14 @@ BlockService.prototype.getBlock = function(blockHash, opts) {
   }
   opts = opts || {};
 
-  var blockData;
   var self = this;
 
   return Promise.try(function() {
-      return self.rpc.getBlockAsync(blockHash);
+      return self.rpc.getBlockAsync(blockHash, false);
     })
     .catch(blockNotFound)
-    .then(function(block) {
-
-      blockData = block.result;
-
-      if (opts.withoutTransactions) {
-        return [];
-      }
-
-      return Promise.all(blockData.tx.map(function(txId) {
-        return self.transactionService.getTransaction(txId);
-      }));
-
-    }).then(function(transactions) {
-
-      blockData.transactions = transactions;
-      return BlockService.blockRPCtoBitcore(blockData);
-
+    .then(function(res) {
+      return bitcore.Block.fromString(res.result);
     });
 };
 
@@ -482,39 +467,61 @@ BlockService.prototype.getBlockchain = function() {
   var blockchain = new BlockChain();
   var headers = [];
 
-  console.log('Fetching headers from db...');
-  var fetchHeader = function(blockHash) {
-    if (blockHash === BlockChain.NULL) {
-      console.log('All headers fetched, total =', headers.length);
-      return;
-    }
-    var headerKey = Index.getBlockHeader(blockHash);
-    return self.database.getAsync(headerKey)
-      .then(function(json) {
-        return bitcore.Block.BlockHeader.fromJSON(json);
-      })
-      .then(function(header) {
-        headers.push(header);
-        return fetchHeader(BufferUtil.reverse(header.prevHash).toString('hex'));
-      });
-  };
+  console.log('Fetching hashes from db...');
 
-  return self._getLatestHash()
-    .then(function(tip) {
-      if (!tip) {
-        console.log('No tip found, syncing blockchain from genesis block');
-        return null;
+  return self.database.getAsync(Index.blockchain, {valueEncoding: 'json'})
+    .catch(function(err) {
+      if(err instanceof LevelUp.errors.NotFoundError) {
+        return [];
       }
-      console.log('Tip is', tip);
-      return fetchHeader(tip)
-        .then(function() {
-          while (headers.length !== 0) {
-            var header = headers.pop();
-            blockchain.proposeNewHeader(header);
+      throw err;
+    })
+    .then(function(blockchainCache) {
+      console.log(blockchainCache.length + ' headers cached');
+
+      var fetchHeader = function(blockHash) {
+        if (blockHash === BlockChain.NULL) {
+          console.log('All headers fetched, total =', headers.length);
+          return Promise.resolve();
+        } else if(blockchainCache.length && blockchainCache[blockchainCache.length - 1].hash === blockHash) {
+          console.log(headers.length + ' non-cached headers loaded');
+          return Promise.resolve();
+        }
+        var headerKey = Index.getBlockHeader(blockHash);
+        return self.database.getAsync(headerKey)
+          .then(function(json) {
+            return bitcore.Block.BlockHeader.fromJSON(json);
+          })
+          .then(function(header) {
+            headers.push(header);
+            return fetchHeader(BufferUtil.reverse(header.prevHash).toString('hex'));
+          });
+      };
+
+      return self._getLatestHash()
+        .then(function(tip) {
+          if (!tip) {
+            console.log('No tip found, syncing blockchain from genesis block');
+            return null;
           }
-          return blockchain;
-        });
+          console.log('Tip is', tip);
+
+          return fetchHeader(tip)
+            .then(function() {
+              blockchain.loadFromCache(blockchainCache);
+
+              while (headers.length !== 0) {
+                var header = headers.pop();
+                blockchain.proposeNewHeader(header);
+              }
+              return blockchain;
+            });
+        })
     });
 };
 
+BlockService.prototype.saveBlockchainCache = function(cache) {
+  return this.database.putAsync(Index.blockchain, JSON.stringify(cache));
+}
+
 module.exports = BlockService;

lib/services/transaction.js

@@ -120,7 +120,7 @@ TransactionService.prototype._confirmOutput = function(ops, block, transaction)
       value: output.toJSON()
     });
     var script = output.script;
-    if (!script.isPublicKeyHashOut() && !script.isScriptHashOut()) {
+    if (!script || !(script.isPublicKeyHashOut() || script.isScriptHashOut())) {
       return;
     }
     var address = output.script.toAddress();
@@ -149,7 +149,7 @@ TransactionService.prototype._confirmInput = function(ops, block, transaction) {
       }))
     });
     var script = input.script;
-    if (!(script.isPublicKeyHashIn() || script.isScriptHashIn())) {
+    if (!script || !(script.isPublicKeyHashIn() || script.isScriptHashIn())) {
       return Promise.resolve();
     }
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "bitcore-node",
   "description": "Full node with extended capabilities using bitcore and Bitcoin Core",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "author": "BitPay <dev@bitpay.com>",
   "repository": "git://github.com/bitpay/bitcore-node.git",
   "contributors": [

test/networkmonitor.js

@@ -79,8 +79,9 @@ describe('NetworkMonitor', function() {
     nm.start();
   });
 
-  it('sends getdatas when receiving invs', function() {
+  it('sends getdatas when receiving invs and ignoreInv is false', function() {
     var nm = new NetworkMonitor(busMock, peerMock);
+    nm.ignoreInv = false;
     nm.start();
     peerMock.sendMessage.calledOnce.should.equal(true);
   });

test/services/block.js

@@ -36,19 +36,7 @@ describe('BlockService', function() {
 
       mockRpc.getBlockAsync = function(block) {
         return Promise.resolve({
-          result: {
-            hash: '000000006a625f06636b8bb6ac7b960a8d03705d1ace08b1a19da3fdcc99ddbd',
-            confirmations: 347064,
-            size: 215,
-            height: 2,
-            version: 1,
-            merkleroot: '9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5',
-            tx: ['9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5'],
-            time: 1231469744,
-            nonce: 1639830024,
-            bits: '1d00ffff',
-            previousblockhash: '00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048'
-          }
+          result: '010000004860eb18bf1b1620e37e9490fc8a427514416fd75159ab86688e9a8300000000d5fdcc541e25de1c7a5addedf24858b8bb665c9f36ef744ee42c316022c90f9bb0bc6649ffff001d08d2bd610101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d010bffffffff0100f2052a010000004341047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77ac00000000'
         });
       };
 

test/services/transaction.js

@@ -85,9 +85,7 @@ describe('TransactionService', function() {
           key: 'txo-4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b-0',
           value: {
             satoshis: 5000000000,
-            script: '65 0x04678afdb0fe5548271967f1a67130b7105cd6a828e03909' +
-              'a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7b' +
-              'a0b8d578a4c702b6bf11d5f OP_CHECKSIG'
+            script: '4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac'
           }
         }]);
         callback();
@@ -113,11 +111,11 @@ describe('TransactionService', function() {
         }, {
           type: 'put',
           key: 'txo-f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16-0',
-          value: '{"satoshis":1000000000,"script":"65 0x04ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84c OP_CHECKSIG"}'
+          value: '{"satoshis":1000000000,"script":"4104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84cac"}'
         }, {
           type: 'put',
           key: 'txo-f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16-1',
-          value: '{"satoshis":4000000000,"script":"65 0x0411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3 OP_CHECKSIG"}'
+          value: '{"satoshis":4000000000,"script":"410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac"}'
         }, {
           type: 'put',
           key: 'txo-f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16-0',