blockstore: index after write interrupt and use less memory

lib/blockstore/file.js

@@ -12,7 +12,7 @@ const assert = require('bsert');
 const fs = require('bfile');
 const bio = require('bufio');
 const Network = require('../protocol/network');
-const Block = require('../primitives/block');
+const Headers = require('../primitives/headers');
 const AbstractBlockStore = require('./abstract');
 const {BlockRecord, FileRecord} = require('./records');
 const layout = require('./layout');
@@ -97,16 +97,29 @@ class FileBlockStore extends AbstractBlockStore {
 
       while (reader.left() >= 4) {
         magic = reader.readU32();
+
+        // Move forward a byte from the last read
+        // if the magic doesn't match.
         if (magic !== this.network.magic) {
-          reader.seek(4);
+          reader.seek(-3);
           continue;
         }
 
         const length = reader.readU32();
         const position = reader.offset;
 
-        const block = Block.fromReader(reader);
-        const hash = block.hash();
+        let header = null;
+
+        try {
+          header = Headers.fromReader(reader);
+          const read = reader.offset - position;
+          reader.seek(length - read);
+        } catch (err) {
+          this.logger.warning(
+            'Unknown block in file: %s, reason: %s',
+            filepath, err.message);
+          continue;
+        }
 
         const blockrecord = new BlockRecord({
           file: fileno,
@@ -114,6 +127,8 @@ class FileBlockStore extends AbstractBlockStore {
           length: length
         });
 
+        const hash = header.hash();
+
         blocks += 1;
         b.put(layout.b.encode(types.BLOCK, hash), blockrecord.toRaw());
       }

test/blockstore-test.js

@@ -18,6 +18,10 @@ const vectors = [
   common.readBlock('block898352')
 ];
 
+const extra = [
+  common.readBlock('block482683')
+];
+
 const {
   AbstractBlockStore,
   FileBlockStore,
@@ -864,6 +868,72 @@ describe('BlockStore', function() {
         assert.bufferEqual(block, expect.block);
       }
     });
+
+    it('will import from files after write interrupt', async () => {
+      const blocks = [];
+
+      for (let i = 0; i < vectors.length; i++) {
+        const [block] = vectors[i].getBlock();
+        const hash = block.hash();
+        const raw = block.toRaw();
+
+        blocks.push({hash, block: raw});
+        await store.write(hash, raw);
+      }
+
+      await store.close();
+
+      assert.equal(await fs.exists(store.filepath(types.BLOCK, 0)), true);
+      assert.equal(await fs.exists(store.filepath(types.BLOCK, 1)), true);
+      assert.equal(await fs.exists(store.filepath(types.BLOCK, 2)), false);
+
+      // Write partial block as would be the case in a
+      // block write interrupt.
+      const [partial] = extra[0].getBlock();
+      {
+        // Include all of the header, but not the block.
+        let raw = partial.toRaw();
+        const actual = raw.length;
+        const part = raw.length - 1;
+        raw = raw.slice(0, part);
+
+        const filepath = store.filepath(types.BLOCK, 1);
+
+        const fd = await fs.open(filepath, 'a');
+
+        const bw = bio.write(8);
+        bw.writeU32(store.network.magic);
+        bw.writeU32(actual);
+        const magic = bw.render();
+
+        const mwritten = await fs.write(fd, magic, 0, 8);
+        const bwritten = await fs.write(fd, raw, 0, part);
+
+        await fs.close(fd);
+
+        assert.equal(mwritten, 8);
+        assert.equal(bwritten, part);
+      }
+
+      await rimraf(resolve(location, './index'));
+
+      store = new FileBlockStore({
+        location: location,
+        maxFileLength: 1024
+      });
+
+      await store.open();
+
+      const incomplete = await store.read(partial.hash());
+      assert(incomplete === null);
+
+      for (let i = 0; i < vectors.length; i++) {
+        const expect = blocks[i];
+        const block = await store.read(expect.hash);
+        assert.equal(block.length, expect.block.length);
+        assert.bufferEqual(block, expect.block);
+      }
+    });
   });
 
   describe('LevelBlockStore', function() {