From 5415147d4e8002227dfd66266c408f6cfb633d69 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Thu, 17 Aug 2017 10:47:20 -0700 Subject: [PATCH] encoding: encoding improvements for base128 varints. --- lib/utils/encoding.js | 55 ++++++++++++++++++++++++++----------------- test/utils-test.js | 24 +++++++++---------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/lib/utils/encoding.js b/lib/utils/encoding.js index 1000f83e..87253d2c 100644 --- a/lib/utils/encoding.js +++ b/lib/utils/encoding.js @@ -12,6 +12,7 @@ */ const {U64, I64} = require('./int64'); +const UINT128_MAX = U64.UINT64_MAX.shrn(7); const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER; const encoding = exports; @@ -345,9 +346,9 @@ encoding.readI53BE = function readI53BE(data, off) { */ encoding._writeI64 = function _writeI64(dst, num, off, be) { - const negative = num < 0; + const neg = num < 0; - if (negative) { + if (neg) { num = -num; num -= 1; } @@ -357,7 +358,7 @@ encoding._writeI64 = function _writeI64(dst, num, off, be) { let hi = (num * (1 / 0x100000000)) | 0; let lo = num | 0; - if (negative) { + if (neg) { hi = ~hi; lo = ~lo; } @@ -491,7 +492,7 @@ encoding.readI64BEN = function readI64BEN(data, off) { */ encoding.writeU64N = function writeU64N(dst, num, off) { - enforce(!num.sign, off, 'Signed.'); + enforce(!num.sign, off, 'Signed'); return num.writeLE(dst, off); }; @@ -504,7 +505,7 @@ encoding.writeU64N = function writeU64N(dst, num, off) { */ encoding.writeU64BEN = function writeU64BEN(dst, num, off) { - enforce(!num.sign, off, 'Signed.'); + enforce(!num.sign, off, 'Signed'); return num.writeBE(dst, off); }; @@ -517,7 +518,7 @@ encoding.writeU64BEN = function writeU64BEN(dst, num, off) { */ encoding.writeI64N = function writeI64N(dst, num, off) { - enforce(num.sign, off, 'Not signed.'); + enforce(num.sign, off, 'Not signed'); return num.writeLE(dst, off); }; @@ -530,7 +531,7 @@ encoding.writeI64N = function writeI64N(dst, num, off) { */ encoding.writeI64BEN = function writeI64BEN(dst, num, off) { - enforce(num.sign, off, 'Not signed.'); + enforce(num.sign, off, 'Not signed'); return num.writeBE(dst, off); }; @@ -682,9 +683,9 @@ encoding.readVarintN = function readVarintN(data, off) { */ encoding.writeVarintN = function writeVarintN(dst, num, off) { - enforce(!num.sign, off, 'Signed.'); + enforce(!num.sign, off, 'Signed'); - if (num.gtn(0xffffffff)) { + if (num.hi !== 0) { dst[off++] = 0xff; return encoding.writeU64N(dst, num, off); } @@ -699,9 +700,9 @@ encoding.writeVarintN = function writeVarintN(dst, num, off) { */ encoding.sizeVarintN = function sizeVarintN(num) { - enforce(!num.sign, 0, 'Signed.'); + enforce(!num.sign, 0, 'Signed'); - if (num.gtn(0xffffffff)) + if (num.hi !== 0) return 9; return encoding.sizeVarint(num.toInt()); @@ -724,13 +725,16 @@ encoding.readVarint2 = function readVarint2(data, off) { const ch = data[off++]; size++; - enforce(num < 0x3fffffffffff, off, 'Number exceeds 2^53-1'); + // Number.MAX_SAFE_INTEGER >>> 7 + enforce(num <= 0x3fffffffffff - (ch & 0x7f), off, 'Number exceeds 2^53-1'); + // num = (num << 7) | (ch & 0x7f); num = (num * 0x80) + (ch & 0x7f); if ((ch & 0x80) === 0) break; + enforce(num !== MAX_SAFE_INTEGER, off, 'Number exceeds 2^53-1'); num++; } @@ -747,17 +751,19 @@ encoding.readVarint2 = function readVarint2(data, off) { encoding.writeVarint2 = function writeVarint2(dst, num, off) { const tmp = []; + let len = 0; for (;;) { tmp[len] = (num & 0x7f) | (len ? 0x80 : 0x00); if (num <= 0x7f) break; + // num = (num >>> 7) - 1; num = ((num - (num % 0x80)) / 0x80) - 1; len++; } - assert(off + len <= dst.length, off); + assert(off + len + 1 <= dst.length, off); do { dst[off++] = tmp[len]; @@ -800,6 +806,7 @@ encoding.sizeVarint2 = function sizeVarint2(num) { size++; if (num <= 0x7f) break; + // num = (num >>> 7) - 1; num = ((num - (num % 0x80)) / 0x80) - 1; } @@ -824,11 +831,14 @@ encoding.readVarint2N = function readVarint2N(data, off) { const ch = data[off++]; size++; - num.iushln(7).iaddn(ch & 0x7f); + enforce(num.lte(UINT128_MAX), off, 'Number exceeds 2^64-1'); + + num.ishln(7).iorn(ch & 0x7f); if ((ch & 0x80) === 0) break; + enforce(!num.eq(U64.UINT64_MAX), off, 'Number exceeds 2^64-1'); num.iaddn(1); } @@ -844,25 +854,26 @@ encoding.readVarint2N = function readVarint2N(data, off) { */ encoding.writeVarint2N = function writeVarint2N(dst, num, off) { - enforce(!num.sign, off, 'Signed.'); + enforce(!num.sign, off, 'Signed'); if (num.hi === 0) return encoding.writeVarint2(dst, num.toInt(), off); - const tmp = []; - let len = 0; - num = num.clone(); + const tmp = []; + + let len = 0; + for (;;) { tmp[len] = num.andln(0x7f) | (len ? 0x80 : 0x00); if (num.lten(0x7f)) break; - num.iushrn(7).isubn(1); + num.ishrn(7).isubn(1); len++; } - enforce(off + len <= dst.length, off, 'Out of bounds write'); + enforce(off + len + 1 <= dst.length, off, 'Out of bounds write'); do { dst[off++] = tmp[len]; @@ -878,7 +889,7 @@ encoding.writeVarint2N = function writeVarint2N(dst, num, off) { */ encoding.sizeVarint2N = function sizeVarint2N(num) { - enforce(!num.sign, 0, 'Signed.'); + enforce(!num.sign, 0, 'Signed'); if (num.hi === 0) return encoding.sizeVarint2(num.toInt()); @@ -891,7 +902,7 @@ encoding.sizeVarint2N = function sizeVarint2N(num) { size++; if (num.lten(0x7f)) break; - num.iushrn(7).isubn(1); + num.ishrn(7).isubn(1); } return size; diff --git a/test/utils-test.js b/test/utils-test.js index 959cceb1..8e7240ac 100644 --- a/test/utils-test.js +++ b/test/utils-test.js @@ -128,54 +128,54 @@ describe('Utils', function() { * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] */ - let b = Buffer.allocUnsafe(1); + let b = Buffer.alloc(1, 0xff); encoding.writeVarint2(b, 0, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 0); assert.deepEqual(b, [0]); - b = Buffer.allocUnsafe(1); + b = Buffer.alloc(1, 0xff); encoding.writeVarint2(b, 1, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 1); assert.deepEqual(b, [1]); - b = Buffer.allocUnsafe(1); + b = Buffer.alloc(1, 0xff); encoding.writeVarint2(b, 127, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 127); assert.deepEqual(b, [0x7f]); - b = Buffer.allocUnsafe(2); + b = Buffer.alloc(2, 0xff); encoding.writeVarint2(b, 128, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 128); assert.deepEqual(b, [0x80, 0x00]); - b = Buffer.allocUnsafe(2); + b = Buffer.alloc(2, 0xff); encoding.writeVarint2(b, 255, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 255); assert.deepEqual(b, [0x80, 0x7f]); - b = Buffer.allocUnsafe(2); + b = Buffer.alloc(2, 0xff); encoding.writeVarint2(b, 16383, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 16383); assert.deepEqual(b, [0xfe, 0x7f]); - b = Buffer.allocUnsafe(2); + b = Buffer.alloc(2, 0xff); encoding.writeVarint2(b, 16384, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 16384); assert.deepEqual(b, [0xff, 0x00]); - b = Buffer.allocUnsafe(3); + b = Buffer.alloc(3, 0xff); encoding.writeVarint2(b, 16511, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 16511); + assert.deepEqual(b.slice(0, 2), [0xff, 0x7f]); // assert.deepEqual(b, [0x80, 0xff, 0x7f]); - assert.deepEqual(b, [0xff, 0x7f, 0x00]); - b = Buffer.allocUnsafe(3); + b = Buffer.alloc(3, 0xff); encoding.writeVarint2(b, 65535, 0); assert.strictEqual(encoding.readVarint2(b, 0).value, 65535); - // assert.deepEqual(b, [0x82, 0xfd, 0x7f]); assert.deepEqual(b, [0x82, 0xfe, 0x7f]); + // assert.deepEqual(b, [0x82, 0xfd, 0x7f]); - b = Buffer.allocUnsafe(5); + b = Buffer.alloc(5, 0xff); encoding.writeVarint2(b, Math.pow(2, 32), 0); assert.strictEqual(encoding.readVarint2(b, 0).value, Math.pow(2, 32)); assert.deepEqual(b, [0x8e, 0xfe, 0xfe, 0xff, 0x00]);