utils: preliminary GCS filter support.

lib/utils/gcs.js

@@ -0,0 +1,578 @@
+/*!
+ * gcs.js - gcs filters for bcoin
+ * Copyright (c) 2017, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+var assert = require('assert');
+var Int64 = require('n64');
+var crypto = require('../crypto/crypto');
+var siphash24 = require('../crypto/siphash');
+var SCRATCH = Buffer.allocUnsafe(64);
+var DUMMY = Buffer.allocUnsafe(0);
+
+/**
+ * GCSFilter
+ * @constructor
+ */
+
+function GCSFilter() {
+  this.n = 0;
+  this.p = 0;
+  this.modp = Int64(0);
+  this.modnp = Int64(0);
+  this.data = DUMMY;
+}
+
+GCSFilter.prototype.hash = function _hash(enc) {
+  var hash = crypto.hash256(this.data);
+  return enc === 'hex' ? hash.toString('hex') : hash;
+};
+
+GCSFilter.prototype.header = function header(prev) {
+  var data = SCRATCH;
+  var hash = this.hash();
+  hash.copy(data, 0);
+  prev.copy(data, 32);
+  return crypto.hash256(data);
+};
+
+GCSFilter.prototype.match = function match(key, data) {
+  var br = new BitReader(this.data);
+  var term = siphash(data, key).imod(this.modnp);
+  var last = Int64(0);
+  var value;
+
+  while (last.lt(term)) {
+    try {
+      value = this.readU64(br);
+    } catch (e) {
+      if (e.message === 'EOF')
+        return false;
+      throw e;
+    }
+
+    value.iadd(last);
+
+    if (value.eq(term))
+      return true;
+
+    last = value;
+  }
+
+  return false;
+};
+
+GCSFilter.prototype.matchAny = function matchAny(key, items) {
+  var br = new BitReader(this.data);
+  var last1 = Int64(0);
+  var values = [];
+  var i, item, hash, last2, cmp, value;
+
+  assert(items.length > 0);
+
+  for (i = 0; i < items.length; i++) {
+    item = items[i];
+    hash = siphash(item, key).imod(this.modnp);
+    values.push(hash);
+  }
+
+  values.sort(compare);
+
+  last2 = values[0];
+  i = 1;
+
+  for (;;) {
+    cmp = last1.cmp(last2);
+
+    if (cmp === 0)
+      break;
+
+    if (cmp > 0) {
+      if (i < values.length) {
+        last2 = values[i];
+        i += 1;
+        continue;
+      }
+      return false;
+    }
+
+    try {
+      value = this.readU64(br);
+    } catch (e) {
+      if (e.message === 'EOF')
+        return false;
+      throw e;
+    }
+
+    last1.iadd(value);
+  }
+
+  return true;
+};
+
+GCSFilter.prototype.readU64 = function readU64(br) {
+  var num = Int64(0);
+  var bit = br.readBit();
+  var rem;
+
+  while (bit) {
+    num.iaddn(1);
+    bit = br.readBit();
+  }
+
+  rem = br.readBits64(this.p);
+
+  return num.imul(this.modp).iadd(rem);
+};
+
+GCSFilter.prototype.toBytes = function toBytes() {
+  return this.data;
+};
+
+GCSFilter.prototype.toNBytes = function toNBytes() {
+  var data = Buffer.allocUnsafe(4 + this.data.length);
+  data.writeUInt32BE(this.n, 0, true);
+  this.data.copy(data, 4);
+  return data;
+};
+
+GCSFilter.prototype.toPBytes = function toPBytes() {
+  var data = Buffer.allocUnsafe(1 + this.data.length);
+  data.writeUInt8(this.p, 0, true);
+  this.data.copy(data, 1);
+  return data;
+};
+
+GCSFilter.prototype.toNPBytes = function toNPBytes() {
+  var data = Buffer.allocUnsafe(5 + this.data.length);
+  data.writeUInt32BE(this.n, 0, true);
+  data.writeUInt8(this.p, 4, true);
+  this.data.copy(data, 5);
+  return data;
+};
+
+GCSFilter.prototype.fromData = function fromData(P, key, items) {
+  var values = [];
+  var last = Int64(0);
+  var i, bw, item, hash, value, rem;
+
+  assert(typeof P === 'number' && isFinite(P));
+  assert(P >= 0 && P <= 32);
+
+  assert(Buffer.isBuffer(key));
+  assert(key.length === 16);
+
+  assert(Array.isArray(items));
+  assert(items.length > 0);
+  assert(items.length <= 0xffffffff);
+
+  this.n = items.length;
+  this.p = P;
+  this.modp = Int64(1).ishln(this.p);
+  this.modnp = Int64(this.n).imul(this.modp);
+
+  bw = new BitWriter();
+
+  for (i = 0; i < items.length; i++) {
+    item = items[i];
+    assert(Buffer.isBuffer(item));
+    hash = siphash(item, key).imod(this.modnp);
+    values.push(hash);
+  }
+
+  values.sort(compare);
+
+  for (i = 0; i < values.length; i++) {
+    hash = values[i];
+    rem = hash.sub(last).iand(this.modp.subn(1));
+    value = hash.sub(last).isub(rem).ishrn(this.p);
+    last = hash;
+
+    // Unary
+    while (!value.isZero()) {
+      bw.writeBit(1);
+      value.isubn(1);
+    }
+    bw.writeBit(0);
+
+    bw.writeBits64(rem, this.p);
+  }
+
+  this.data = bw.render();
+
+  return this;
+};
+
+GCSFilter.prototype.fromBytes = function fromBytes(N, P, data) {
+  assert(typeof N === 'number' && isFinite(N));
+  assert(typeof P === 'number' && isFinite(P));
+  assert(P >= 0 && P <= 32);
+  assert(Buffer.isBuffer(data));
+
+  this.n = N;
+  this.p = P;
+  this.modp = Int64(1).ishln(this.p);
+  this.modnp = Int64(this.n).imul(this.modp);
+  this.data = data;
+
+  return this;
+};
+
+GCSFilter.prototype.fromNBytes = function fromNBytes(P, data) {
+  var N;
+
+  assert(typeof P === 'number' && isFinite(P));
+  assert(Buffer.isBuffer(data));
+  assert(data.length >= 4);
+
+  N = data.readUInt32BE(0, true);
+
+  return this.fromBytes(N, P, data.slice(4));
+};
+
+GCSFilter.prototype.fromPBytes = function fromPBytes(N, data) {
+  var P;
+
+  assert(typeof N === 'number' && isFinite(N));
+  assert(Buffer.isBuffer(data));
+  assert(data.length >= 1);
+
+  P = data.readUInt8(0, true);
+
+  return this.fromBytes(N, P, data.slice(1));
+};
+
+GCSFilter.prototype.fromNPBytes = function fromNPBytes(data) {
+  var N, P;
+
+  assert(Buffer.isBuffer(data));
+  assert(data.length >= 5);
+
+  N = data.readUInt32BE(0, true);
+  P = data.readUInt8(4, true);
+
+  return this.fromBytes(N, P, data.slice(5));
+};
+
+GCSFilter.prototype.fromBlock = function fromBlock(block) {
+  var hash = block.hash();
+  var key = hash.slice(0, 16);
+  var items = [];
+  var i, j, tx, input, output;
+
+  for (i = 0; i < block.txs.length; i++) {
+    tx = block.txs[i];
+
+    if (i > 0) {
+      for (j = 0; j < tx.inputs.length; j++) {
+        input = tx.inputs[j];
+        items.push(input.prevout.toRaw());
+      }
+    }
+
+    for (j = 0; j < tx.outputs.length; j++) {
+      output = tx.outputs[j];
+      getPushes(items, output.script);
+    }
+  }
+
+  return this.fromData(20, key, items);
+};
+
+GCSFilter.prototype.fromExtended = function fromExtended(block) {
+  var hash = block.hash();
+  var key = hash.slice(0, 16);
+  var items = [];
+  var i, j, tx, input;
+
+  for (i = 0; i < block.txs.length; i++) {
+    tx = block.txs[i];
+
+    items.push(tx.hash());
+
+    if (i > 0) {
+      for (j = 0; j < tx.inputs.length; j++) {
+        input = tx.inputs[j];
+        getWitness(items, input.witness);
+        getPushes(items, input.script);
+      }
+    }
+  }
+
+  return this.fromData(20, key, items);
+};
+
+GCSFilter.fromData = function fromData(P, key, items) {
+  return new GCSFilter().fromData(P, key, items);
+};
+
+GCSFilter.fromBytes = function fromBytes(N, P, data) {
+  return new GCSFilter().fromBytes(N, P, data);
+};
+
+GCSFilter.fromNBytes = function fromNBytes(P, data) {
+  return new GCSFilter().fromNBytes(P, data);
+};
+
+GCSFilter.fromPBytes = function fromPBytes(N, data) {
+  return new GCSFilter().fromPBytes(N, data);
+};
+
+GCSFilter.fromNPBytes = function fromNPBytes(data) {
+  return new GCSFilter().fromNPBytes(data);
+};
+
+GCSFilter.fromBlock = function fromBlock(block) {
+  return new GCSFilter().fromBlock(block);
+};
+
+GCSFilter.fromExtended = function fromExtended(block) {
+  return new GCSFilter().fromExtended(block);
+};
+
+/**
+ * BitWriter
+ * @constructor
+ */
+
+function BitWriter() {
+  this.stream = [];
+  this.remain = 0;
+}
+
+BitWriter.prototype.writeBit = function writeBit(bit) {
+  var index;
+
+  if (this.remain === 0) {
+    this.stream.push(0);
+    this.remain = 8;
+  }
+
+  if (bit) {
+    index = this.stream.length - 1;
+    this.stream[index] |= 1 << (this.remain - 1);
+  }
+
+  this.remain--;
+};
+
+BitWriter.prototype.writeOneByte = function writeOneByte(ch) {
+  var index;
+
+  if (this.remain === 0) {
+    this.stream.push(0);
+    this.remain = 8;
+  }
+
+  index = this.stream.length - 1;
+
+  this.stream[index] |= ch >> (8 - this.remain);
+  this.stream.push(0);
+  this.stream[index + 1] = ch << this.remain;
+};
+
+BitWriter.prototype.writeBits = function writeBits(num, count) {
+  var ch, bit;
+
+  assert(count >= 0);
+  assert(count <= 32);
+
+  num <<= 32 - count;
+
+  while (count >= 8) {
+    ch = num >>> 24;
+    this.writeOneByte(ch);
+
+    num <<= 8;
+    count -= 8;
+  }
+
+  while (count > 0) {
+    bit = num >>> 31;
+    this.writeBit(bit);
+    num <<= 1;
+    count--;
+  }
+};
+
+BitWriter.prototype.writeBits64 = function writeBits64(num, count) {
+  if (count > 32) {
+    this.writeBits(num.hi, count - 32);
+    this.writeBits(num.lo, 32);
+  } else {
+    this.writeBits(num.lo, count);
+  }
+};
+
+BitWriter.prototype.render = function render() {
+  var stream = this.stream;
+  var data = Buffer.allocUnsafe(stream.length);
+  var i;
+
+  for (i = 0; i < stream.length; i++)
+    data[i] = stream[i];
+
+  return data;
+};
+
+/**
+ * BitReader
+ * @constructor
+ */
+
+function BitReader(data) {
+  this.stream = copy(data);
+  this.pos = 0;
+  this.remain = 8;
+}
+
+BitReader.prototype.readBit = function readBit() {
+  var bit;
+
+  if (this.pos >= this.stream.length)
+    throw new Error('EOF');
+
+  if (this.remain === 0) {
+    this.pos += 1;
+
+    if (this.pos >= this.stream.length)
+      throw new Error('EOF');
+
+    this.remain = 8;
+  }
+
+  bit = this.stream[this.pos] & 0x80;
+
+  this.stream[this.pos] <<= 1;
+  this.stream[this.pos] &= 0xff;
+  this.remain--;
+
+  return bit !== 0 ? 1 : 0;
+};
+
+BitReader.prototype.readByte = function readByte() {
+  var ch;
+
+  if (this.pos >= this.stream.length)
+    throw new Error('EOF');
+
+  if (this.remain === 0) {
+    this.pos += 1;
+
+    if (this.pos >= this.stream.length)
+      throw new Error('EOF');
+
+    this.remain = 8;
+  }
+
+  if (this.remain === 8) {
+    ch = this.stream[this.pos];
+    this.pos += 1;
+    return ch;
+  }
+
+  ch = this.stream[this.pos];
+  this.pos += 1;
+
+  if (this.pos >= this.stream.length)
+    throw new Error('EOF');
+
+  ch |= this.stream[this.pos] >> this.remain;
+
+  this.stream[this.pos] <<= (8 - this.remain);
+  this.stream[this.pos] &= 0xff;
+
+  return ch;
+};
+
+BitReader.prototype.readBits = function readBits(count) {
+  var num = 0;
+  var ch, bit;
+
+  assert(count >= 0);
+  assert(count <= 32);
+
+  while (count >= 8) {
+    num <<= 8;
+    ch = this.readByte();
+    num |= ch;
+    count -= 8;
+  }
+
+  while (count > 0) {
+    num <<= 1;
+    bit = this.readBit();
+    if (bit)
+      num |= 1;
+    count -= 1;
+  }
+
+  return num;
+};
+
+BitReader.prototype.readBits64 = function readBits(count) {
+  var n = new Int64();
+
+  if (count > 32) {
+    n.hi = this.readBits(count - 32);
+    n.lo = this.readBits(32);
+  } else {
+    n.lo = this.readBits(count);
+  }
+
+  return n;
+};
+
+/*
+ * Helpers
+ */
+
+function compare(a, b) {
+  return a.cmp(b) < 0 ? -1 : 1;
+}
+
+function siphash(data, key) {
+  var hash = siphash24(data, key);
+  return Int64().join(hash.hi, hash.lo);
+}
+
+function copy(data) {
+  var clone = Buffer.allocUnsafe(data.length);
+  data.copy(clone, 0, 0, data.length);
+  return clone;
+}
+
+function getPushes(items, script) {
+  var i, op;
+
+  for (i = 0; i < script.code.length; i++) {
+    op = script.code[i];
+
+    if (!op.data || op.data.length === 0)
+      continue;
+
+    items.push(op.data);
+  }
+}
+
+function getWitness(items, witness) {
+  var i, data;
+
+  for (i = 0; i < witness.items.length; i++) {
+    data = witness.items[i];
+
+    if (data.length === 0)
+      continue;
+
+    items.push(data);
+  }
+}
+
+/*
+ * Expose
+ */
+
+module.exports = GCSFilter;

package.json

@@ -25,7 +25,8 @@
   "dependencies": {
     "bn.js": "4.11.6",
     "elliptic": "6.4.0",
-    "hmac-drbg": "^1.0.0"
+    "hmac-drbg": "^1.0.0",
+    "n64": "0.0.5"
   },
   "optionalDependencies": {
     "bcoin-native": "0.0.18",

test/gcs-test.js

@@ -0,0 +1,176 @@
+'use strict';
+
+var assert = require('assert');
+var fs = require('fs');
+var GCSFilter = require('../lib/utils/gcs');
+var util = require('../lib/utils/util');
+var crypto = require('../lib/crypto/crypto');
+var Block = require('../lib/primitives/block');
+var Outpoint = require('../lib/primitives/outpoint');
+var Address = require('../lib/primitives/address');
+
+var raw = fs.readFileSync(__dirname + '/data/block928927.raw');
+var block = Block.fromRaw(raw);
+
+describe('GCS', function() {
+  var key = crypto.randomBytes(16);
+  var P = 20;
+  var filter1, filter2, filter3, filter4, filter5;
+  var contents1, contents2;
+  var op1, op2, op3, op4;
+  var addr1, addr2;
+
+  contents1 = [
+    Buffer.from('Alex', 'ascii'),
+    Buffer.from('Bob', 'ascii'),
+    Buffer.from('Charlie', 'ascii'),
+    Buffer.from('Dick', 'ascii'),
+    Buffer.from('Ed', 'ascii'),
+    Buffer.from('Frank', 'ascii'),
+    Buffer.from('George', 'ascii'),
+    Buffer.from('Harry', 'ascii'),
+    Buffer.from('Ilya', 'ascii'),
+    Buffer.from('John', 'ascii'),
+    Buffer.from('Kevin', 'ascii'),
+    Buffer.from('Larry', 'ascii'),
+    Buffer.from('Michael', 'ascii'),
+    Buffer.from('Nate', 'ascii'),
+    Buffer.from('Owen', 'ascii'),
+    Buffer.from('Paul', 'ascii'),
+    Buffer.from('Quentin', 'ascii')
+  ];
+
+  contents2 = [
+    Buffer.from('Alice', 'ascii'),
+    Buffer.from('Betty', 'ascii'),
+    Buffer.from('Charmaine', 'ascii'),
+    Buffer.from('Donna', 'ascii'),
+    Buffer.from('Edith', 'ascii'),
+    Buffer.from('Faina', 'ascii'),
+    Buffer.from('Georgia', 'ascii'),
+    Buffer.from('Hannah', 'ascii'),
+    Buffer.from('Ilsbeth', 'ascii'),
+    Buffer.from('Jennifer', 'ascii'),
+    Buffer.from('Kayla', 'ascii'),
+    Buffer.from('Lena', 'ascii'),
+    Buffer.from('Michelle', 'ascii'),
+    Buffer.from('Natalie', 'ascii'),
+    Buffer.from('Ophelia', 'ascii'),
+    Buffer.from('Peggy', 'ascii'),
+    Buffer.from('Queenie', 'ascii')
+  ];
+
+  op1 = new Outpoint(
+    '4cba1d1753ed19dbeafffb1a6c805d20e4af00b194a8f85353163cef83319c2c',
+    4);
+
+  op2 = new Outpoint(
+    'b7c3c4bce1a23baef2da05f9b7e4bff813449ec7e80f980ec7e4cacfadcd3314',
+    3);
+
+  op3 = new Outpoint(
+    '4cba1d1753ed19dbeafffb1a6c805d20e4af00b194a8f85353163cef83319c2c',
+    400);
+
+  op4 = new Outpoint(
+    'b7c3c4bce1a23baef2da05f9b7e4bff813449ec7e80f980ec7e4cacfadcd3314',
+    300);
+
+  addr1 = new Address('bc1qmyrddmxglk49ye2wd29wefaavw7es8k5d555lx');
+  addr2 = new Address('bc1q4645ycu0l9pnvxaxnhemushv0w4cd9flkqh95j');
+
+  it('should test GCS filter build', function() {
+    filter1 = GCSFilter.fromData(P, key, contents1);
+    assert(filter1);
+  });
+
+  it('should test GCS filter copy', function() {
+    filter2 = GCSFilter.fromBytes(filter1.n, P, filter1.toBytes());
+    assert(filter2);
+    filter3 = GCSFilter.fromNBytes(P, filter1.toNBytes());
+    assert(filter3);
+    filter4 = GCSFilter.fromPBytes(filter1.n, filter1.toPBytes());
+    assert(filter4);
+    filter5 = GCSFilter.fromNPBytes(filter1.toNPBytes());
+    assert(filter5);
+  });
+
+  it('should test GCS filter metadata', function() {
+    assert.equal(filter1.p, P);
+    assert.equal(filter1.n, contents1.length);
+    assert.equal(filter1.p, filter2.p);
+    assert.equal(filter1.n, filter2.n);
+    assert.deepEqual(filter1.data, filter2.data);
+    assert.equal(filter1.p, filter3.p);
+    assert.equal(filter1.n, filter3.n);
+    assert.deepEqual(filter1.data, filter3.data);
+    assert.equal(filter1.p, filter4.p);
+    assert.equal(filter1.n, filter4.n);
+    assert.deepEqual(filter1.data, filter4.data);
+    assert.equal(filter1.p, filter5.p);
+    assert.equal(filter1.n, filter5.n);
+    assert.deepEqual(filter1.data, filter5.data);
+  });
+
+  it('should test GCS filter match', function() {
+    var match = filter1.match(key, Buffer.from('Nate'));
+    assert(match);
+    match = filter2.match(key, Buffer.from('Nate'));
+    assert(match);
+    match = filter1.match(key, Buffer.from('Quentin'));
+    assert(match);
+    match = filter2.match(key, Buffer.from('Quentin'));
+    assert(match);
+
+    match = filter1.match(key, Buffer.from('Nates'));
+    assert(!match);
+    match = filter2.match(key, Buffer.from('Nates'));
+    assert(!match);
+    match = filter1.match(key, Buffer.from('Quentins'));
+    assert(!match);
+    match = filter2.match(key, Buffer.from('Quentins'));
+    assert(!match);
+  });
+
+  it('should test GCS filter matchAny', function() {
+    var c, match;
+
+    match = filter1.matchAny(key, contents2);
+    assert(!match);
+    match = filter2.matchAny(key, contents2);
+    assert(!match);
+
+    c = contents2.slice();
+    c.push(Buffer.from('Nate'));
+
+    match = filter1.matchAny(key, c);
+    assert(match);
+    match = filter2.matchAny(key, c);
+    assert(match);
+  });
+
+  it('should test GCS filter fromBlock', function() {
+    var key = block.hash().slice(0, 16);
+    var filter = GCSFilter.fromBlock(block);
+    assert(filter.match(key, op1.toRaw()));
+    assert(filter.match(key, op2.toRaw()));
+    assert(!filter.match(key, op3.toRaw()));
+    assert(!filter.match(key, op4.toRaw()));
+    assert(filter.match(key, addr1.hash));
+    assert(filter.match(key, addr2.hash));
+    assert(filter.matchAny(key, [op1.toRaw(), addr1.hash]));
+    assert(filter.matchAny(key, [op1.toRaw(), op3.toRaw()]));
+    assert(!filter.matchAny(key, [op3.toRaw(), op4.toRaw()]));
+  });
+
+  it('should test GCS filter fromExtended', function() {
+    var key = block.hash().slice(0, 16);
+    var filter = GCSFilter.fromExtended(block);
+    assert(!filter.match(key, op1.toRaw()));
+    assert(filter.match(key, block.txs[0].hash()));
+    assert(filter.match(key, block.txs[1].hash()));
+    assert(filter.matchAny(key, [block.txs[0].hash(), block.txs[1].hash()]));
+    assert(filter.matchAny(key, [op1.toRaw(), block.txs[1].hash()]));
+    assert(!filter.matchAny(key, [op1.toRaw(), op2.toRaw()]));
+  });
+});