serve merkleblocks.

lib/bcoin/merkleblock.js

@@ -187,6 +187,95 @@ MerkleBlock.isMerkleBlock = function isMerkleBlock(obj) {
     && typeof obj._verifyPartial === 'function';
 };
 
+MerkleBlock.prototype.fromBlock = function fromBlock(block, bloom) {
+  var matches = [];
+  var txs = [];
+  var leaves = [];
+  var bits = [];
+  var hashes = [];
+  var i, tx, totalTX, height, flags, p;
+
+  for (i = 0; i < block.txs.length; i++) {
+    tx = block.txs[i];
+    if (tx.isWatched(bloom)) {
+      matches.push(1);
+      txs.push(tx);
+    } else {
+      matches.push(0);
+    }
+    leaves.push(tx.hash());
+  }
+
+  totalTX = leaves.length;
+
+  function width(height) {
+    return (totalTX + (1 << height) - 1) >> height;
+  }
+
+  function hash(height, pos, leaves) {
+    var left, right;
+
+    if (height === 0)
+      return leaves[0];
+
+    left = hash(height - 1, pos * 2, leaves);
+
+    if (pos * 2 + 1 < width(height - 1, pos * 2 + 1, leaves))
+      right = hash(height - 1, pos * 2 + 1, leaves);
+    else
+      right = left;
+
+    return utils.dsha256(Buffer.concat([left, right]));
+  }
+
+  function traverse(height, pos, leaves, matches) {
+    var parent = 0;
+    var p;
+
+    for (p = pos << height; p < (pos + 1) << height && p < totalTX; p++)
+      parent |= matches[p];
+
+    bits.push(parent);
+
+    if (height === 0 || !parent) {
+      hashes.push(hash(height, pos, leaves));
+      return;
+    }
+
+    traverse(height - 1, pos * 2, leaves, matches);
+
+    if (pos * 2 + 1 < width(height - 1))
+      traverse(height - 1, pos * 2 + 1, leaves, matches);
+  }
+
+  height = 0;
+  while (width(height) > 1)
+    height++;
+
+  traverse(height, 0, leaves, matches);
+
+  flags = new Buffer((bits.length + 7) / 8 | 0);
+  for (p = 0; p < bits.length; p++)
+    flags[p / 8 | 0] |= bits[p] << (p % 8);
+
+  block = new MerkleBlock({
+    version: block.version,
+    prevBlock: block.prevBlock,
+    merkleRoot: block.merkleRoot,
+    ts: block.ts,
+    bits: block.bits,
+    nonce: block.nonce,
+    totalTX: totalTX,
+    height: block.height,
+    hashes: hashes,
+    flags: flags
+  });
+
+  block.txs = txs;
+
+  return block;
+};
+
 /**
  * Expose
  */

lib/bcoin/miner.js

@@ -246,7 +246,7 @@ Miner.prototype.createBlock = function createBlock(callback) {
       headers = {
         version: version,
         prevBlock: self.chain.tip.hash,
-        merkleRoot: utils.toHex(constants.zeroHash),
+        merkleRoot: constants.zeroHash,
         ts: ts,
         bits: target,
         nonce: 0
@@ -254,12 +254,12 @@ Miner.prototype.createBlock = function createBlock(callback) {
 
       block = bcoin.block(headers);
 
-      block.addTX(coinbase);
-
       block.height = self.chain.height + 1;
       block.target = utils.fromCompact(target).toBuffer('le', 32);
       block.extraNonce = new bn(0);
 
+      block.addTX(coinbase);
+
       if (self.chain.segwitActive) {
         // Set up the witness nonce and
         // commitment output for segwit.

lib/bcoin/peer.js

@@ -46,6 +46,7 @@ function Peer(pool, options) {
   this.sendHeaders = false;
   this.haveWitness = false;
   this.hashContinue = null;
+  this.filter = null;
 
   this.challenge = null;
   this.lastPong = 0;
@@ -228,6 +229,9 @@ Peer.prototype.broadcast = function broadcast(items) {
   var result = [];
   var payload = [];
 
+  if (this.version && this.version.relay === false)
+    return;
+
   if (this.destroyed)
     return;
 
@@ -240,11 +244,6 @@ Peer.prototype.broadcast = function broadcast(items) {
     var type = item.type;
     var entry, packetType;
 
-    if (old) {
-      clearTimeout(old.timer);
-      clearInterval(old.interval);
-    }
-
     if (typeof type === 'string')
       type = constants.inv[type];
 
@@ -261,6 +260,16 @@ Peer.prototype.broadcast = function broadcast(items) {
     else
       assert(false, 'Bad type.');
 
+    if (self.filter && type === constants.inv.tx) {
+      if (!item.isWatched(self.filter))
+        return;
+    }
+
+    if (old) {
+      clearTimeout(old.timer);
+      clearInterval(old.interval);
+    }
+
     // Auto-cleanup broadcast map after timeout
     entry = {
       e: new EventEmitter(),
@@ -452,6 +461,12 @@ Peer.prototype._onPacket = function onPacket(packet) {
       return this._handleGetHeaders(payload);
     case 'mempool':
       return this._handleMempool(payload);
+    case 'filterload':
+      return this._handleFilterLoad(payload);
+    case 'filteradd':
+      return this._handleFilterAdd(payload);
+    case 'filterclear':
+      return this._handleFilterClear(payload);
     case 'block':
       payload = bcoin.compactblock(payload);
       this._emit(cmd, payload);
@@ -502,6 +517,31 @@ Peer.prototype._emitMerkle = function _emitMerkle() {
   this.lastBlock = null;
 };
 
+Peer.prototype._handleFilterLoad = function _handleFilterLoad(payload) {
+  var size = payload.filter.length * 8;
+  this.filter = new bcoin.bloom(size, payload.n, payload.tweak);
+  this.filter.filter = payload.filter;
+  this.filter.update = payload.update;
+  if (this.version)
+    this.version.relay = true;
+};
+
+Peer.prototype._handleFilterAdd = function _handleFilterAdd(payload) {
+  if (this.filter)
+    this.filter.add(payload.data);
+
+  if (this.version)
+    this.version.relay = true;
+};
+
+Peer.prototype._handleFilterClear = function _handleFilterClear(payload) {
+  if (this.filter)
+    this.filter.reset();
+
+  if (this.version)
+    this.version.relay = true;
+};
+
 Peer.prototype._handleUTXOs = function _handleUTXOs(payload) {
   payload.coins = payload.coins(function(coin) {
     return new bcoin.coin(coin);
@@ -515,6 +555,12 @@ Peer.prototype._handleGetUTXOs = function _handleGetUTXOs(payload) {
   var coins = [];
   var notfound = [];
 
+  if (this.pool.options.selfish)
+    return;
+
+  if (this.chain.db.options.spv)
+    return;
+
   function checkMempool(hash, index, callback) {
     if (!self.mempool)
       return callback();
@@ -595,6 +641,9 @@ Peer.prototype._handleGetHeaders = function _handleGetHeaders(payload) {
   if (this.chain.db.options.spv)
     return;
 
+  if (this.chain.db.prune)
+    return;
+
   function collect(err, hash) {
     if (err)
       return done(err);
@@ -660,6 +709,9 @@ Peer.prototype._handleGetBlocks = function _handleGetBlocks(payload) {
   if (this.chain.db.options.spv)
     return;
 
+  if (this.chain.db.prune)
+    return;
+
   function done(err) {
     if (err)
       return self.emit('error', err);
@@ -828,7 +880,7 @@ Peer.prototype._handleGetData = function handleGetData(items) {
 
     if (type === constants.inv.tx) {
       if (!self.mempool) {
-        notfound.push({ type: type, hash: hash });
+        notfound.push({ type: constants.inv.tx, hash: hash });
         return next();
       }
       return self.mempool.getTX(hash, function(err, tx) {
@@ -836,7 +888,7 @@ Peer.prototype._handleGetData = function handleGetData(items) {
           return next(err);
 
         if (!tx) {
-          notfound.push({ type: type, hash: hash });
+          notfound.push({ type: constants.inv.tx, hash: hash });
           return next();
         }
 
@@ -853,15 +905,19 @@ Peer.prototype._handleGetData = function handleGetData(items) {
 
     if (type === constants.inv.block) {
       if (self.chain.db.options.spv) {
-        notfound.push({ type: type, hash: hash });
+        notfound.push({ type: constants.inv.block, hash: hash });
         return next();
       }
+      if (self.chain.db.prune) {
+        notfound.push({ type: constants.inv.block, hash: hash });
+        return;
+      }
       return self.chain.db.getBlock(hash, function(err, block) {
         if (err)
           return next(err);
 
         if (!block) {
-          notfound.push({ type: type, hash: hash });
+          notfound.push({ type: constants.inv.block, hash: hash });
           return next();
         }
 
@@ -874,7 +930,50 @@ Peer.prototype._handleGetData = function handleGetData(items) {
 
         if (hash === self.hashContinue) {
           self._write(self.framer.inv([{
-            type: type,
+            type: constants.inv.block,
+            hash: self.chain.tip.hash
+          }]));
+          self.hashContinue = null;
+        }
+
+        next();
+      });
+    }
+
+    if (type === constants.inv.filteredblock) {
+      if (self.chain.db.options.spv) {
+        notfound.push({ type: constants.inv.block, hash: hash });
+        return next();
+      }
+      if (self.chain.db.prune) {
+        notfound.push({ type: constants.inv.block, hash: hash });
+        return;
+      }
+      return self.chain.db.getBlock(hash, function(err, block) {
+        if (err)
+          return next(err);
+
+        if (!block) {
+          notfound.push({ type: constants.inv.block, hash: hash });
+          return next();
+        }
+
+        block = bcoin.merkleblock.fromBlock(block, self.filter);
+
+        self._write(self.framer.merkleBlock(block));
+
+        block.txs.forEach(function(tx) {
+          if (isWitness)
+            tx = tx.renderWitness();
+          else
+            tx = tx.renderNormal();
+
+          self._write(self.framer.packet('tx', tx));
+        });
+
+        if (hash === self.hashContinue) {
+          self._write(self.framer.inv([{
+            type: constants.inv.block,
             hash: self.chain.tip.hash
           }]));
           self.hashContinue = null;

lib/bcoin/pool.js

@@ -1223,63 +1223,6 @@ Pool.prototype.updateWatch = function updateWatch() {
   });
 };
 
-// See "Filter matching algorithm":
-// https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki
-Pool.prototype.isWatched = function(tx, bloom) {
-  var i, input, output;
-
-  if (!bloom)
-    bloom = this.bloom;
-
-  function testScript(code) {
-    return code.some(function(chunk) {
-      if (!Buffer.isBuffer(chunk) || chunk.length === 0)
-        return false;
-      return bloom.test(chunk);
-    });
-  }
-
-  // 1. Test the tx hash
-  if (bloom.test(tx.hash()))
-    return true;
-
-  // 2. Test data elements in output scripts
-  //    (may need to update filter on match)
-  for (i = 0; i < tx.outputs.length; i++) {
-    output = tx.outputs[i];
-    // Test the output script
-    if (testScript(output.script.code))
-      return true;
-  }
-
-  // 3. Test prev_out structure
-  // 4. Test data elements in input scripts
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-
-    // Test the prev_out tx hash
-    if (bloom.test(input.prevout.hash, 'hex'))
-      return true;
-
-    // Test the prev_out script
-    if (input.coin) {
-      if (testScript(input.coin.script.code))
-        return true;
-    }
-
-    // Test the input script
-    if (testScript(input.script.code))
-      return true;
-
-    // Test the witness
-    if (testScript(input.witness.items))
-      return true;
-  }
-
-  // 5. No match
-  return false;
-};
-
 Pool.prototype.addWallet = function addWallet(wallet, callback) {
   var self = this;
 

lib/bcoin/protocol/constants.js

@@ -42,6 +42,8 @@ exports.filterFlags = {
   pubkeyOnly: 2
 };
 
+exports.filterFlagsByVal = utils.revMap(exports.filterFlags);
+
 exports.opcodes = {
   OP_FALSE: 0x00,
   OP_0: 0x00,

lib/bcoin/protocol/framer.js

@@ -319,11 +319,29 @@ Framer.pong = function pong(data) {
 
 Framer.filterLoad = function filterLoad(bloom, update, writer) {
   var p = new BufferWriter(writer);
+  var filter, n, tweak;
 
-  p.writeVarBytes(bloom.toBuffer());
+  if (bloom instanceof bcoin.bloom) {
-  p.writeU32(bloom.n);
+    filter = bloom.toBuffer();
-  p.writeU32(bloom.tweak);
+    n = bloom.n;
-  p.writeU8(constants.filterFlags[update]);
+    tweak = bloom.tweak;
+  } else {
+    writer = update;
+    update = bloom.update;
+    filter = bloom.filter;
+    n = bloom.n;
+    tweak = bloom.tweak;
+  }
+
+  if (typeof update === 'string')
+    update = constants.filterFlags[update];
+
+  assert(update != null, 'Bad filter flag.');
+
+  p.writeVarBytes(filter);
+  p.writeU32(n);
+  p.writeU32(tweak);
+  p.writeU8(update);
 
   if (!writer)
     p = p.render();
@@ -444,6 +462,18 @@ Framer.tx = function _tx(tx, writer) {
   return p;
 };
 
+Framer.outpoint = function outpoint(hash, index, writer) {
+  var p = new BufferWriter(writer);
+
+  p.writeHash(hash);
+  p.writeU32(index);
+
+  if (!writer)
+    p = p.render();
+
+  return p;
+};
+
 Framer.input = function _input(input, writer) {
   var p = new BufferWriter(writer);
 

lib/bcoin/protocol/parser.js

@@ -163,7 +163,25 @@ Parser.parseGetAddr = function parseGetAddr(p) {
 };
 
 Parser.parseFilterLoad = function parseFilterLoad(p) {
-  return {};
+  var filter, n, tweak, update;
+
+  p = new BufferReader(p);
+  p.start();
+
+  filter = p.readVarBytes();
+  n = p.readU32();
+  tweak = p.readU32();
+  update = constants.filterFlagsByVal[p.readU8()];
+
+  assert(update != null, 'Bad filter flag.');
+
+  return {
+    filter: filter,
+    n: n,
+    tweak: tweak,
+    update: update,
+    _size: p.end()
+  };
 };
 
 Parser.parseFilterAdd = function parseFilterAdd(p) {

lib/bcoin/tx.js

@@ -1126,6 +1126,70 @@ TX.prototype.hasType = function hasType(type) {
   return false;
 };
 
+// See "Filter matching algorithm":
+// https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki
+TX.prototype.isWatched = function isWatched(bloom) {
+  var i, input, output, hash, index, outpoint;
+
+  if (!bloom)
+    return false;
+
+  function testScript(code) {
+    return code.some(function(chunk) {
+      if (!Buffer.isBuffer(chunk) || chunk.length === 0)
+        return false;
+      return bloom.test(chunk);
+    });
+  }
+
+  // 1. Test the tx hash
+  if (bloom.test(this.hash()))
+    return true;
+
+  // 2. Test data elements in output scripts
+  //    (may need to update filter on match)
+  for (i = 0; i < this.outputs.length; i++) {
+    output = this.outputs[i];
+    // Test the output script
+    if (testScript(output.script.code)) {
+      if (bloom.update === 'all') {
+        outpoint = bcoin.protocol.framer.outpoint(this.hash(), i);
+        bloom.add(outpoint);
+      } else if (bloom.update === 'pubkeyOnly') {
+        if (output.script.isPubkey() || output.script.isMultisig()) {
+          outpoint = bcoin.protocol.framer.outpoint(this.hash(), i);
+          bloom.add(outpoint);
+        }
+      }
+      return true;
+    }
+  }
+
+  // 3. Test prev_out structure
+  // 4. Test data elements in input scripts
+  for (i = 0; i < this.inputs.length; i++) {
+    input = this.inputs[i];
+    hash = input.prevout.hash;
+    index = input.prevout.index;
+    outpoint = bcoin.protocol.framer.outpoint(hash, index);
+
+    // Test the COutPoint structure
+    if (bloom.test(outpoint))
+      return true;
+
+    // Test the input script
+    if (testScript(input.script.code))
+      return true;
+
+    // Test the witness
+    if (testScript(input.witness.items))
+      return true;
+  }
+
+  // 5. No match
+  return false;
+};
+
 TX.prototype.__defineGetter__('rblock', function() {
   return this.block
     ? utils.revHex(this.block)