indexer: add block position to tx

lib/indexer/indexer.js

@@ -533,6 +533,8 @@ class Indexer extends EventEmitter {
    */
 
   async _addBlock(entry, block, view) {
+    assert(block.hasRaw(), 'Expected raw data for block.');
+
     const tip = BlockMeta.fromEntry(entry);
 
     if (tip.height >= this.network.block.slowHeight && !this.rescanning)

lib/indexer/txindexer.js

@@ -131,17 +131,11 @@ class TXIndexer extends Indexer {
   async indexBlock(entry, block, view) {
     const b = this.db.batch();
 
-    const data = block.toRaw();
-    const br = bio.read(data);
-    // ignore header
-    br.readBytes(80);
-    const count = br.readVarint();
+    for (let i = 0; i < block.txs.length; i++) {
+      const tx = block.txs[i];
 
-    for (let i = 0; i < count; i++) {
-      const offset = br.offset;
-      const tx = TX.fromReader(br);
-      const length = br.offset - offset;
       const hash = tx.hash();
+      const {offset, size} = tx.getPosition();
 
       const txrecord = new TxRecord({
         block: entry.hash,
@@ -149,8 +143,9 @@ class TXIndexer extends Indexer {
         time: entry.time,
         index: i,
         offset: offset,
-        length: length
+        length: size
       });
+
       b.put(layout.t.encode(hash), txrecord.toRaw());
     }
 

lib/primitives/block.js

@@ -109,6 +109,15 @@ class Block extends AbstractBlock {
     return this.frame().data;
   }
 
+  /**
+   * Check if block has been serialized.
+   * @returns {Buffer}
+   */
+
+  hasRaw() {
+    return Boolean(this._raw);
+  }
+
   /**
    * Serialize the block, do not include witnesses.
    * @returns {Buffer}
@@ -645,7 +654,7 @@ class Block extends AbstractBlock {
     let witness = 0;
 
     for (let i = 0; i < count; i++) {
-      const tx = TX.fromReader(br);
+      const tx = TX.fromReader(br, true);
       witness += tx._witness;
       this.txs.push(tx);
     }
@@ -738,7 +747,7 @@ class Block extends AbstractBlock {
     bw.writeVarint(this.txs.length);
 
     for (const tx of this.txs)
-      tx.toWriter(bw);
+      tx.toWriter(bw, true);
 
     return bw;
   }

lib/primitives/tx.js

@@ -57,6 +57,8 @@ class TX {
     this._whash = null;
 
     this._raw = null;
+    this._offset = -1;
+    this._block = false;
     this._size = -1;
     this._witness = -1;
     this._sigops = -1;
@@ -157,6 +159,8 @@ class TX {
 
     this._raw = null;
     this._size = -1;
+    this._offset = -1;
+    this._block = false;
     this._witness = -1;
     this._sigops = -1;
 
@@ -245,15 +249,21 @@ class TX {
   /**
    * Write the transaction to a buffer writer.
    * @param {BufferWriter} bw
+   * @param {Boolean} block
    */
 
-  toWriter(bw) {
+  toWriter(bw, block) {
     if (this.mutable) {
       if (this.hasWitness())
         return this.writeWitness(bw);
       return this.writeNormal(bw);
     }
 
+    if (block) {
+      this._offset = bw.offset;
+      this._block = true;
+    }
+
     bw.writeBytes(this.toRaw());
 
     return bw;
@@ -311,6 +321,21 @@ class TX {
     return raw;
   }
 
+  /**
+   * Return the offset and size of the transaction. Useful
+   * when the transaction is deserialized within a block.
+   * @returns {Object} Contains `size` and `offset`.
+   */
+
+  getPosition() {
+    assert(this._block && this._offset > 80, 'Position not available.');
+
+    return {
+      offset: this._offset,
+      size: this._size
+    };
+  }
+
   /**
    * Calculate total size and size of the witness bytes.
    * @returns {Object} Contains `size` and `witness`.
@@ -2226,11 +2251,12 @@ class TX {
   /**
    * Instantiate a transaction from a buffer reader.
    * @param {BufferReader} br
+   * @param {Boolean} block
    * @returns {TX}
    */
 
-  static fromReader(br) {
-    return new this().fromReader(br);
+  static fromReader(br, block) {
+    return new this().fromReader(br, block);
   }
 
   /**
@@ -2247,13 +2273,14 @@ class TX {
    * Inject properties from buffer reader.
    * @private
    * @param {BufferReader} br
+   * @param {Boolean} block
    */
 
-  fromReader(br) {
+  fromReader(br, block) {
     if (hasWitnessBytes(br))
-      return this.fromWitnessReader(br);
+      return this.fromWitnessReader(br, block);
 
-    br.start();
+    const start = br.start();
 
     this.version = br.readU32();
 
@@ -2269,6 +2296,11 @@ class TX {
 
     this.locktime = br.readU32();
 
+    if (block) {
+      this._offset = start;
+      this._block = true;
+    }
+
     if (!this.mutable) {
       this._raw = br.endData();
       this._size = this._raw.length;
@@ -2285,10 +2317,11 @@ class TX {
    * buffer reader (witness serialization).
    * @private
    * @param {BufferReader} br
+   * @param {Boolean} block
    */
 
-  fromWitnessReader(br) {
-    br.start();
+  fromWitnessReader(br, block) {
+    const start = br.start();
 
     this.version = br.readU32();
 
@@ -2336,6 +2369,11 @@ class TX {
 
     this.locktime = br.readU32();
 
+    if (block) {
+      this._offset = start;
+      this._block = true;
+    }
+
     if (!this.mutable && hasWitness) {
       this._raw = br.endData();
       this._size = this._raw.length;

test/block-test.js

@@ -375,4 +375,91 @@ describe('Block', function() {
       });
     }
   }
+
+  it('should deserialize with offset positions for txs (witness)', () => {
+    const [block] = block482683.getBlock();
+
+    const expected = [
+      {offset: 81, size: 217},
+      {offset: 298, size: 815},
+      {offset: 1113, size: 192},
+      {offset: 1305, size: 259},
+      {offset: 1564, size: 223},
+      {offset: 1787, size: 1223},
+      {offset: 3010, size: 486},
+      {offset: 3496, size: 665},
+      {offset: 4161, size: 3176},
+      {offset: 7337, size: 225},
+      {offset: 7562, size: 1223},
+      {offset: 8785, size: 503}
+    ];
+
+    assert.equal(expected.length, block.txs.length);
+    assert.equal(block.getSize(), expected.reduce((a, b) => a + b.size, 81));
+
+    for (let i = 0; i < block.txs.length; i++) {
+      const {offset, size} = block.txs[i].getPosition();
+
+      assert.strictEqual(offset, expected[i].offset);
+      assert.strictEqual(size, expected[i].size);
+    }
+  });
+
+  it('should serialize with offset positions for txs (witness)', () => {
+    const [block] = block482683.getBlock();
+
+    const expected = [
+      {offset: 81, size: 217},
+      {offset: 298, size: 815},
+      {offset: 1113, size: 192},
+      {offset: 1305, size: 259},
+      {offset: 1564, size: 223},
+      {offset: 1787, size: 1223},
+      {offset: 3010, size: 486},
+      {offset: 3496, size: 665},
+      {offset: 4161, size: 3176},
+      {offset: 7337, size: 225},
+      {offset: 7562, size: 1223},
+      {offset: 8785, size: 503}
+    ];
+
+    assert.equal(expected.length, block.txs.length);
+    assert.equal(block.getSize(), expected.reduce((a, b) => a + b.size, 81));
+
+    // Reset the offset for all transactions, and clear
+    // any cached values for the block.
+    block.refresh(true);
+    for (let i = 0; i < block.txs.length; i++)
+      assert.equal(block.txs[i]._offset, -1);
+
+    // Serialize the block, as done before saving to disk.
+    const raw = block.toRaw();
+    assert(raw);
+
+    for (let i = 0; i < block.txs.length; i++) {
+      const {offset, size} = block.txs[i].getPosition();
+
+      assert.strictEqual(offset, expected[i].offset);
+      assert.strictEqual(size, expected[i].size);
+    }
+  });
+
+  it('should deserialize with offset positions for txs', () => {
+    const [block] = block300025.getBlock();
+
+    assert.equal(block.txs.length, 461);
+
+    let expect = 83;
+    let total = 83;
+
+    for (let i = 0; i < block.txs.length; i++) {
+      const {offset, size} = block.txs[i].getPosition();
+
+      assert.strictEqual(offset, expect);
+      expect += size;
+      total += size;
+    }
+
+    assert.equal(total, 284231);
+  });
 });

test/indexer-test.js

@@ -11,6 +11,7 @@ const Miner = require('../lib/mining/miner');
 const MemWallet = require('./util/memwallet');
 const TXIndexer = require('../lib/indexer/txindexer');
 const AddrIndexer = require('../lib/indexer/addrindexer');
+const BlockStore = require('../lib/blockstore/level');
 const Network = require('../lib/protocol/network');
 const network = Network.get('regtest');
 
@@ -18,10 +19,16 @@ const workers = new WorkerPool({
   enabled: true
 });
 
+const blocks = new BlockStore({
+  memory: true,
+  network
+});
+
 const chain = new Chain({
   memory: true,
   network,
-  workers
+  workers,
+  blocks
 });
 
 const miner = new Miner({
@@ -52,6 +59,7 @@ describe('Indexer', function() {
   this.timeout(45000);
 
   it('should open indexer', async () => {
+    await blocks.open();
     await chain.open();
     await miner.open();
     await txindexer.open();