diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index b894d225..87356ad8 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -533,6 +533,8 @@ class Indexer extends EventEmitter { */ async _addBlock(entry, block, view) { + assert(block.hasRaw(), 'Expected raw data for block.'); + const tip = BlockMeta.fromEntry(entry); if (tip.height >= this.network.block.slowHeight && !this.rescanning) diff --git a/lib/indexer/txindexer.js b/lib/indexer/txindexer.js index b09d48a2..8f68d6b4 100644 --- a/lib/indexer/txindexer.js +++ b/lib/indexer/txindexer.js @@ -131,17 +131,11 @@ class TXIndexer extends Indexer { async indexBlock(entry, block, view) { const b = this.db.batch(); - const data = block.toRaw(); - const br = bio.read(data); - // ignore header - br.readBytes(80); - const count = br.readVarint(); + for (let i = 0; i < block.txs.length; i++) { + const tx = block.txs[i]; - for (let i = 0; i < count; i++) { - const offset = br.offset; - const tx = TX.fromReader(br); - const length = br.offset - offset; const hash = tx.hash(); + const {offset, size} = tx.getPosition(); const txrecord = new TxRecord({ block: entry.hash, @@ -149,8 +143,9 @@ class TXIndexer extends Indexer { time: entry.time, index: i, offset: offset, - length: length + length: size }); + b.put(layout.t.encode(hash), txrecord.toRaw()); } diff --git a/lib/primitives/block.js b/lib/primitives/block.js index b10b3dc9..f0766f62 100644 --- a/lib/primitives/block.js +++ b/lib/primitives/block.js @@ -109,6 +109,15 @@ class Block extends AbstractBlock { return this.frame().data; } + /** + * Check if block has been serialized. + * @returns {Buffer} + */ + + hasRaw() { + return Boolean(this._raw); + } + /** * Serialize the block, do not include witnesses. * @returns {Buffer} @@ -645,7 +654,7 @@ class Block extends AbstractBlock { let witness = 0; for (let i = 0; i < count; i++) { - const tx = TX.fromReader(br); + const tx = TX.fromReader(br, true); witness += tx._witness; this.txs.push(tx); } @@ -738,7 +747,7 @@ class Block extends AbstractBlock { bw.writeVarint(this.txs.length); for (const tx of this.txs) - tx.toWriter(bw); + tx.toWriter(bw, true); return bw; } diff --git a/lib/primitives/tx.js b/lib/primitives/tx.js index d3ead244..bfc94b11 100644 --- a/lib/primitives/tx.js +++ b/lib/primitives/tx.js @@ -57,6 +57,8 @@ class TX { this._whash = null; this._raw = null; + this._offset = -1; + this._block = false; this._size = -1; this._witness = -1; this._sigops = -1; @@ -157,6 +159,8 @@ class TX { this._raw = null; this._size = -1; + this._offset = -1; + this._block = false; this._witness = -1; this._sigops = -1; @@ -245,15 +249,21 @@ class TX { /** * Write the transaction to a buffer writer. * @param {BufferWriter} bw + * @param {Boolean} block */ - toWriter(bw) { + toWriter(bw, block) { if (this.mutable) { if (this.hasWitness()) return this.writeWitness(bw); return this.writeNormal(bw); } + if (block) { + this._offset = bw.offset; + this._block = true; + } + bw.writeBytes(this.toRaw()); return bw; @@ -311,6 +321,21 @@ class TX { return raw; } + /** + * Return the offset and size of the transaction. Useful + * when the transaction is deserialized within a block. + * @returns {Object} Contains `size` and `offset`. + */ + + getPosition() { + assert(this._block && this._offset > 80, 'Position not available.'); + + return { + offset: this._offset, + size: this._size + }; + } + /** * Calculate total size and size of the witness bytes. * @returns {Object} Contains `size` and `witness`. @@ -2226,11 +2251,12 @@ class TX { /** * Instantiate a transaction from a buffer reader. * @param {BufferReader} br + * @param {Boolean} block * @returns {TX} */ - static fromReader(br) { - return new this().fromReader(br); + static fromReader(br, block) { + return new this().fromReader(br, block); } /** @@ -2247,13 +2273,14 @@ class TX { * Inject properties from buffer reader. * @private * @param {BufferReader} br + * @param {Boolean} block */ - fromReader(br) { + fromReader(br, block) { if (hasWitnessBytes(br)) - return this.fromWitnessReader(br); + return this.fromWitnessReader(br, block); - br.start(); + const start = br.start(); this.version = br.readU32(); @@ -2269,6 +2296,11 @@ class TX { this.locktime = br.readU32(); + if (block) { + this._offset = start; + this._block = true; + } + if (!this.mutable) { this._raw = br.endData(); this._size = this._raw.length; @@ -2285,10 +2317,11 @@ class TX { * buffer reader (witness serialization). * @private * @param {BufferReader} br + * @param {Boolean} block */ - fromWitnessReader(br) { - br.start(); + fromWitnessReader(br, block) { + const start = br.start(); this.version = br.readU32(); @@ -2336,6 +2369,11 @@ class TX { this.locktime = br.readU32(); + if (block) { + this._offset = start; + this._block = true; + } + if (!this.mutable && hasWitness) { this._raw = br.endData(); this._size = this._raw.length; diff --git a/test/block-test.js b/test/block-test.js index 2ff7848a..20f2047b 100644 --- a/test/block-test.js +++ b/test/block-test.js @@ -375,4 +375,91 @@ describe('Block', function() { }); } } + + it('should deserialize with offset positions for txs (witness)', () => { + const [block] = block482683.getBlock(); + + const expected = [ + {offset: 81, size: 217}, + {offset: 298, size: 815}, + {offset: 1113, size: 192}, + {offset: 1305, size: 259}, + {offset: 1564, size: 223}, + {offset: 1787, size: 1223}, + {offset: 3010, size: 486}, + {offset: 3496, size: 665}, + {offset: 4161, size: 3176}, + {offset: 7337, size: 225}, + {offset: 7562, size: 1223}, + {offset: 8785, size: 503} + ]; + + assert.equal(expected.length, block.txs.length); + assert.equal(block.getSize(), expected.reduce((a, b) => a + b.size, 81)); + + for (let i = 0; i < block.txs.length; i++) { + const {offset, size} = block.txs[i].getPosition(); + + assert.strictEqual(offset, expected[i].offset); + assert.strictEqual(size, expected[i].size); + } + }); + + it('should serialize with offset positions for txs (witness)', () => { + const [block] = block482683.getBlock(); + + const expected = [ + {offset: 81, size: 217}, + {offset: 298, size: 815}, + {offset: 1113, size: 192}, + {offset: 1305, size: 259}, + {offset: 1564, size: 223}, + {offset: 1787, size: 1223}, + {offset: 3010, size: 486}, + {offset: 3496, size: 665}, + {offset: 4161, size: 3176}, + {offset: 7337, size: 225}, + {offset: 7562, size: 1223}, + {offset: 8785, size: 503} + ]; + + assert.equal(expected.length, block.txs.length); + assert.equal(block.getSize(), expected.reduce((a, b) => a + b.size, 81)); + + // Reset the offset for all transactions, and clear + // any cached values for the block. + block.refresh(true); + for (let i = 0; i < block.txs.length; i++) + assert.equal(block.txs[i]._offset, -1); + + // Serialize the block, as done before saving to disk. + const raw = block.toRaw(); + assert(raw); + + for (let i = 0; i < block.txs.length; i++) { + const {offset, size} = block.txs[i].getPosition(); + + assert.strictEqual(offset, expected[i].offset); + assert.strictEqual(size, expected[i].size); + } + }); + + it('should deserialize with offset positions for txs', () => { + const [block] = block300025.getBlock(); + + assert.equal(block.txs.length, 461); + + let expect = 83; + let total = 83; + + for (let i = 0; i < block.txs.length; i++) { + const {offset, size} = block.txs[i].getPosition(); + + assert.strictEqual(offset, expect); + expect += size; + total += size; + } + + assert.equal(total, 284231); + }); }); diff --git a/test/indexer-test.js b/test/indexer-test.js index a6b4ee93..b7a61047 100644 --- a/test/indexer-test.js +++ b/test/indexer-test.js @@ -11,6 +11,7 @@ const Miner = require('../lib/mining/miner'); const MemWallet = require('./util/memwallet'); const TXIndexer = require('../lib/indexer/txindexer'); const AddrIndexer = require('../lib/indexer/addrindexer'); +const BlockStore = require('../lib/blockstore/level'); const Network = require('../lib/protocol/network'); const network = Network.get('regtest'); @@ -18,10 +19,16 @@ const workers = new WorkerPool({ enabled: true }); +const blocks = new BlockStore({ + memory: true, + network +}); + const chain = new Chain({ memory: true, network, - workers + workers, + blocks }); const miner = new Miner({ @@ -52,6 +59,7 @@ describe('Indexer', function() { this.timeout(45000); it('should open indexer', async () => { + await blocks.open(); await chain.open(); await miner.open(); await txindexer.open();