blockstore: add block data types with an undo type

This commit is contained in:
Braydon Fuller 2019-03-06 11:27:41 -08:00
parent abd2ae4b5d
commit 3cec13ef5e
No known key found for this signature in database
GPG Key ID: F24F232D108B3AD4
6 changed files with 409 additions and 65 deletions

View File

@ -51,10 +51,16 @@ class AbstractBlockStore {
}
/**
* This method stores block data. The action should be idempotent.
* If the data is already stored, the behavior will be the same. Any
* concurrent requests to store the same data will produce the same
* result, and will not conflict with each other.
* This method stores block undo coin data.
* @returns {Promise}
*/
async writeUndo(hash, data) {
throw new Error('Abstract method.');
}
/**
* This method stores block data.
* @returns {Promise}
*/
@ -62,6 +68,15 @@ class AbstractBlockStore {
throw new Error('Abstract method.');
}
/**
* This method will retrieve block undo coin data.
* @returns {Promise}
*/
async readUndo(hash) {
throw new Error('Abstract method.');
}
/**
* This method will retrieve block data. Smaller portions of
* the block can be read by using the offset and size arguments.
@ -73,9 +88,16 @@ class AbstractBlockStore {
}
/**
* This will free resources for storing the block data. This
* may not mean that the block is deleted, but that it should
* no longer consume any local storage resources.
* This will free resources for storing the block undo coin data.
* @returns {Promise}
*/
async pruneUndo(hash) {
throw new Error('Abstract method.');
}
/**
* This will free resources for storing the block data.
* @returns {Promise}
*/
@ -83,6 +105,16 @@ class AbstractBlockStore {
throw new Error('Abstract method.');
}
/**
* This will check if a block undo coin data has been stored
* and is available.
* @returns {Promise}
*/
async hasUndo(hash) {
throw new Error('Abstract method.');
}
/**
* This will check if a block has been stored and is available.
* @returns {Promise}

31
lib/blockstore/common.js Normal file
View File

@ -0,0 +1,31 @@
/*!
* common.js - block store constants for bcoin
* Copyright (c) 2019, Braydon Fuller (MIT License).
* https://github.com/bcoin-org/bcoin
*/
'use strict';
/**
* @module blockstore/common
*/
/**
* Data types.
* @enum {Number}
*/
exports.types = {
BLOCK: 1,
UNDO: 2
};
/**
* File prefixes for data types.
* @enum {String}
*/
exports.prefixes = {
1: 'blk',
2: 'blu'
};

View File

@ -16,6 +16,7 @@ const Block = require('../primitives/block');
const AbstractBlockStore = require('./abstract');
const {BlockRecord, FileRecord} = require('./records');
const layout = require('./layout');
const {types, prefixes} = require('./common');
/**
* File Block Store
@ -66,7 +67,7 @@ class FileBlockStore extends AbstractBlockStore {
let missing = false;
for (const fileno of filenos) {
const rec = await this.db.get(layout.f.encode(fileno));
const rec = await this.db.get(layout.f.encode(types.BLOCK, fileno));
if (!rec) {
missing = true;
break;
@ -80,7 +81,7 @@ class FileBlockStore extends AbstractBlockStore {
for (const fileno of filenos) {
const b = this.db.batch();
const filepath = this.filepath(fileno);
const filepath = this.filepath(types.BLOCK, fileno);
const data = await fs.readFile(filepath);
const reader = bio.read(data);
let magic = null;
@ -106,7 +107,7 @@ class FileBlockStore extends AbstractBlockStore {
});
blocks += 1;
b.put(layout.b.encode(hash), blockrecord.toRaw());
b.put(layout.b.encode(types.BLOCK, hash), blockrecord.toRaw());
}
const filerecord = new FileRecord({
@ -115,7 +116,7 @@ class FileBlockStore extends AbstractBlockStore {
length: this.maxFileLength
});
b.put(layout.f.encode(fileno), filerecord.toRaw());
b.put(layout.f.encode(types.BLOCK, fileno), filerecord.toRaw());
await b.write();
@ -152,11 +153,12 @@ class FileBlockStore extends AbstractBlockStore {
/**
* This method will determine the file path based on the file number
* and the current block data location.
* @private
* @param {Number} fileno - The number of the file.
* @returns {Promise}
*/
filepath(fileno) {
filepath(type, fileno) {
const pad = 5;
let num = fileno.toString(10);
@ -167,17 +169,27 @@ class FileBlockStore extends AbstractBlockStore {
while (num.length < pad)
num = `0${num}`;
return join(this.location, `blk${num}.dat`);
let filepath = null;
const prefix = prefixes[type];
if (!prefix)
throw new Error('Unknown file prefix.');
filepath = join(this.location, `${prefix}${num}.dat`);
return filepath;
}
/**
* This method will select and potentially allocate a file to
* write a block based on the size.
* @private
* @param {Number} length - The number of bytes of the data to be written.
* @returns {Promise}
*/
async allocate(length) {
async allocate(type, length) {
if (length > this.maxFileLength)
throw new Error('Block length above max file length.');
@ -185,13 +197,13 @@ class FileBlockStore extends AbstractBlockStore {
let filerecord = null;
let filepath = null;
const last = await this.db.get(layout.R.encode());
const last = await this.db.get(layout.F.encode(type));
if (last)
fileno = bio.read(last).readU32();
filepath = this.filepath(fileno);
filepath = this.filepath(type, fileno);
const rec = await this.db.get(layout.f.encode(fileno));
const rec = await this.db.get(layout.f.encode(type, fileno));
let touch = false;
@ -208,7 +220,7 @@ class FileBlockStore extends AbstractBlockStore {
if (filerecord.used + length > filerecord.length) {
fileno += 1;
filepath = this.filepath(fileno);
filepath = this.filepath(type, fileno);
touch = true;
filerecord = new FileRecord({
blocks: 0,
@ -225,6 +237,17 @@ class FileBlockStore extends AbstractBlockStore {
return {fileno, filerecord, filepath};
}
/**
* This method stores block undo coin data in files.
* @param {Buffer} hash - The block hash
* @param {Buffer} data - The block data
* @returns {Promise}
*/
async writeUndo(hash, data) {
return this._write(types.UNDO, hash, data);
}
/**
* This method stores block data in files.
* @param {Buffer} hash - The block hash
@ -233,6 +256,20 @@ class FileBlockStore extends AbstractBlockStore {
*/
async write(hash, data) {
return this._write(types.BLOCK, hash, data);
}
/**
* This method stores block data in files with by appending
* data to the last written file and updating indexes to point
* to the file and position.
* @private
* @param {Buffer} hash - The block hash
* @param {Buffer} data - The block data
* @returns {Promise}
*/
async _write(type, hash, data) {
if (this.writing)
throw new Error('Already writing.');
@ -246,7 +283,7 @@ class FileBlockStore extends AbstractBlockStore {
fileno,
filerecord,
filepath
} = await this.allocate(length);
} = await this.allocate(type, length);
const mposition = filerecord.used;
const bposition = filerecord.used + mlength;
@ -280,17 +317,27 @@ class FileBlockStore extends AbstractBlockStore {
length: blength
});
b.put(layout.b.encode(hash), blockrecord.toRaw());
b.put(layout.f.encode(fileno), filerecord.toRaw());
b.put(layout.b.encode(type, hash), blockrecord.toRaw());
b.put(layout.f.encode(type, fileno), filerecord.toRaw());
const bw = bio.write(4);
b.put(layout.R.encode(), bw.writeU32(fileno).render());
const last = bio.write(4).writeU32(fileno).render();
b.put(layout.F.encode(type), last);
await b.write();
this.writing = false;
}
/**
* This method will retrieve block undo coin data.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async readUndo(hash) {
return this._read(types.UNDO, hash);
}
/**
* This method will retrieve block data. Smaller portions of the
* block (e.g. transactions) can be read by using the offset and
@ -302,13 +349,28 @@ class FileBlockStore extends AbstractBlockStore {
*/
async read(hash, offset, length) {
const raw = await this.db.get(layout.b.encode(hash));
return this._read(types.BLOCK, hash, offset, length);
}
/**
* This methods reads data from disk by retrieving the index of
* the data and reading from the correponding file and location.
* @private
* @param {Buffer} type - The data type
* @param {Buffer} hash - The block hash
* @param {Number} offset - The offset within the block
* @param {Number} length - The number of bytes of the data
* @returns {Promise}
*/
async _read(type, hash, offset, length) {
const raw = await this.db.get(layout.b.encode(type, hash));
if (!raw)
return null;
const blockrecord = BlockRecord.fromRaw(raw);
const filepath = this.filepath(blockrecord.file);
const filepath = this.filepath(type, blockrecord.file);
let position = blockrecord.position;
@ -331,22 +393,43 @@ class FileBlockStore extends AbstractBlockStore {
}
/**
* This will free resources for storing the block data. The block
* data may not be deleted from disk immediately, the index for
* the block is removed and will not be able to be read. The underlying
* file is unlinked when all blocks in a file have been pruned.
* This will free resources for storing the block undo coin data.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async pruneUndo(hash) {
return this._prune(types.UNDO, hash);
}
/**
* This will free resources for storing the block data.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async prune(hash) {
const braw = await this.db.get(layout.b.encode(hash));
return this._prune(types.BLOCK, hash);
}
/**
* This will free resources for storing the block data. The block
* data may not be deleted from disk immediately, the index for the
* block is removed and will not be able to be read. The underlying
* file is unlinked when all blocks in a file have been pruned.
* @private
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async _prune(type, hash) {
const braw = await this.db.get(layout.b.encode(type, hash));
if (!braw)
return false;
const blockrecord = BlockRecord.fromRaw(braw);
const fraw = await this.db.get(layout.f.encode(blockrecord.file));
const fraw = await this.db.get(layout.f.encode(type, blockrecord.file));
if (!fraw)
return false;
@ -357,20 +440,31 @@ class FileBlockStore extends AbstractBlockStore {
const b = this.db.batch();
if (filerecord.blocks === 0)
b.del(layout.f.encode(blockrecord.file));
b.del(layout.f.encode(type, blockrecord.file));
else
b.put(layout.f.encode(blockrecord.file), filerecord.toRaw());
b.put(layout.f.encode(type, blockrecord.file), filerecord.toRaw());
b.del(layout.b.encode(hash));
b.del(layout.b.encode(type, hash));
await b.write();
if (filerecord.blocks === 0)
await fs.unlink(this.filepath(blockrecord.file));
await fs.unlink(this.filepath(type, blockrecord.file));
return true;
}
/**
* This will check if a block undo coin has been stored
* and is available.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async hasUndo(hash) {
return await this.db.has(layout.b.encode(types.UNDO, hash));
}
/**
* This will check if a block has been stored and is available.
* @param {Buffer} hash - The block hash
@ -378,7 +472,7 @@ class FileBlockStore extends AbstractBlockStore {
*/
async has(hash) {
return await this.db.has(layout.b.encode(hash));
return await this.db.has(layout.b.encode(types.BLOCK, hash));
}
}

View File

@ -11,16 +11,16 @@ const bdb = require('bdb');
/*
* Database Layout:
* V -> db version
* R -> last file entry
* f[uint32] -> file entry
* b[hash] -> block entry
* B[type] -> last file record by type
* f[type][fileno] -> file record by type and file number
* b[type][hash] -> block record by type and block hash
*/
const layout = {
V: bdb.key('V'),
R: bdb.key('R'),
f: bdb.key('f', ['uint32']),
b: bdb.key('b', ['hash256'])
F: bdb.key('F', ['uint32']),
f: bdb.key('f', ['uint32', 'uint32']),
b: bdb.key('b', ['uint32', 'hash256'])
};
/*

View File

@ -11,6 +11,7 @@ const bdb = require('bdb');
const assert = require('bsert');
const AbstractBlockStore = require('./abstract');
const layout = require('./layout');
const {types} = require('./common');
/**
* LevelDB Block Store
@ -58,6 +59,17 @@ class LevelBlockStore extends AbstractBlockStore {
await this.db.close();
}
/**
* This method stores block undo coin data in LevelDB.
* @param {Buffer} hash - The block hash
* @param {Buffer} data - The block data
* @returns {Promise}
*/
async writeUndo(hash, data) {
return this.db.put(layout.b.encode(types.UNDO, hash), data);
}
/**
* This method stores block data in LevelDB.
* @param {Buffer} hash - The block hash
@ -66,7 +78,17 @@ class LevelBlockStore extends AbstractBlockStore {
*/
async write(hash, data) {
return this.db.put(layout.b.encode(hash), data);
return this.db.put(layout.b.encode(types.BLOCK, hash), data);
}
/**
* This method will retrieve block undo coin data.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async readUndo(hash) {
return this.db.get(layout.b.encode(types.UNDO, hash));
}
/**
@ -81,7 +103,7 @@ class LevelBlockStore extends AbstractBlockStore {
*/
async read(hash, offset, length) {
let raw = await this.db.get(layout.b.encode(hash));
let raw = await this.db.get(layout.b.encode(types.BLOCK, hash));
if (offset) {
if (offset + length > raw.length)
@ -93,6 +115,23 @@ class LevelBlockStore extends AbstractBlockStore {
return raw;
}
/**
* This will free resources for storing the block undo coin data.
* The block data may not be immediately removed from disk, and will
* be reclaimed during LevelDB compaction.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async pruneUndo(hash) {
if (!await this.hasUndo(hash))
return false;
await this.db.del(layout.b.encode(types.UNDO, hash));
return true;
}
/**
* This will free resources for storing the block data. The block
* data may not be immediately removed from disk, and will be reclaimed
@ -105,11 +144,22 @@ class LevelBlockStore extends AbstractBlockStore {
if (!await this.has(hash))
return false;
await this.db.del(layout.b.encode(hash));
await this.db.del(layout.b.encode(types.BLOCK, hash));
return true;
}
/**
* This will check if a block undo coin data has been stored
* and is available.
* @param {Buffer} hash - The block hash
* @returns {Promise}
*/
async hasUndo(hash) {
return this.db.has(layout.b.encode(types.UNDO, hash));
}
/**
* This will check if a block has been stored and is available.
* @param {Buffer} hash - The block hash
@ -117,7 +167,7 @@ class LevelBlockStore extends AbstractBlockStore {
*/
async has(hash) {
return this.db.has(layout.b.encode(hash));
return this.db.has(layout.b.encode(types.BLOCK, hash));
}
}

View File

@ -26,6 +26,7 @@ const {
} = require('../lib/blockstore');
const layout = require('../lib/blockstore/layout');
const {types} = require('../lib/blockstore/common');
const {
BlockRecord,
@ -281,7 +282,7 @@ describe('BlockStore', function() {
it('will fail with length above file max', async () => {
let err = null;
try {
await store.allocate(1025);
await store.allocate(types.BLOCK, 1025);
} catch (e) {
err = e;
}
@ -292,39 +293,39 @@ describe('BlockStore', function() {
describe('filepath', function() {
it('will give correct path (0)', () => {
const filepath = store.filepath(0);
const filepath = store.filepath(types.BLOCK, 0);
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00000.dat');
});
it('will give correct path (1)', () => {
const filepath = store.filepath(7);
const filepath = store.filepath(types.BLOCK, 7);
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00007.dat');
});
it('will give correct path (2)', () => {
const filepath = store.filepath(23);
const filepath = store.filepath(types.BLOCK, 23);
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00023.dat');
});
it('will give correct path (3)', () => {
const filepath = store.filepath(456);
const filepath = store.filepath(types.BLOCK, 456);
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00456.dat');
});
it('will give correct path (4)', () => {
const filepath = store.filepath(8999);
const filepath = store.filepath(types.BLOCK, 8999);
assert.equal(filepath, '/tmp/.bcoin/blocks/blk08999.dat');
});
it('will give correct path (5)', () => {
const filepath = store.filepath(99999);
const filepath = store.filepath(types.BLOCK, 99999);
assert.equal(filepath, '/tmp/.bcoin/blocks/blk99999.dat');
});
it('will fail over max size', () => {
let err = null;
try {
store.filepath(100000);
store.filepath(types.BLOCK, 100000);
} catch (e) {
err = e;
}
@ -332,6 +333,11 @@ describe('BlockStore', function() {
assert(err);
assert.equal(err.message, 'File number too large.');
});
it('will give undo type', () => {
const filepath = store.filepath(types.UNDO, 99999);
assert.equal(filepath, '/tmp/.bcoin/blocks/blu99999.dat');
});
});
});
@ -366,6 +372,17 @@ describe('BlockStore', function() {
assert.bufferEqual(block1, block2);
});
it('will write and read block undo coins', async () => {
const block1 = random.randomBytes(128);
const hash = random.randomBytes(32);
await store.writeUndo(hash, block1);
const block2 = await store.readUndo(hash);
assert.bufferEqual(block1, block2);
});
it('will read a block w/ offset and length', async () => {
const block1 = random.randomBytes(128);
const hash = random.randomBytes(32);
@ -412,9 +429,9 @@ describe('BlockStore', function() {
assert.bufferEqual(block2, block);
}
const first = await fs.stat(store.filepath(0));
const second = await fs.stat(store.filepath(1));
const third = await fs.stat(store.filepath(2));
const first = await fs.stat(store.filepath(types.BLOCK, 0));
const second = await fs.stat(store.filepath(types.BLOCK, 1));
const third = await fs.stat(store.filepath(types.BLOCK, 2));
assert.equal(first.size, 952);
assert.equal(second.size, 952);
assert.equal(third.size, 272);
@ -429,6 +446,35 @@ describe('BlockStore', function() {
}
});
it('will allocate new files with block undo coins', async () => {
const blocks = [];
for (let i = 0; i < 16; i++) {
const block = random.randomBytes(128);
const hash = random.randomBytes(32);
blocks.push({hash, block});
await store.writeUndo(hash, block);
const block2 = await store.readUndo(hash);
assert.bufferEqual(block2, block);
}
const first = await fs.stat(store.filepath(types.UNDO, 0));
const second = await fs.stat(store.filepath(types.UNDO, 1));
const third = await fs.stat(store.filepath(types.UNDO, 2));
assert.equal(first.size, 952);
assert.equal(second.size, 952);
assert.equal(third.size, 272);
const len = first.size + second.size + third.size - (8 * 16);
assert.equal(len, 128 * 16);
for (let i = 0; i < 16; i++) {
const expect = blocks[i];
const block = await store.readUndo(expect.hash);
assert.bufferEqual(block, expect.block);
}
});
it('will recover from interrupt during block write', async () => {
{
const block = random.randomBytes(128);
@ -445,7 +491,7 @@ describe('BlockStore', function() {
// would not be updated to include the used bytes and
// thus this data should be overwritten.
{
const filepath = store.filepath(0);
const filepath = store.filepath(types.BLOCK, 0);
const fd = await fs.open(filepath, 'a');
@ -522,6 +568,20 @@ describe('BlockStore', function() {
assert.strictEqual(exists, true);
});
it('will check if block undo coins exists (false)', async () => {
const hash = random.randomBytes(32);
const exists = await store.hasUndo(hash);
assert.strictEqual(exists, false);
});
it('will check if block undo coins exists (true)', async () => {
const block = random.randomBytes(128);
const hash = random.randomBytes(32);
await store.writeUndo(hash, block);
const exists = await store.hasUndo(hash);
assert.strictEqual(exists, true);
});
it('will prune blocks', async () => {
const hashes = [];
for (let i = 0; i < 16; i++) {
@ -531,9 +591,9 @@ describe('BlockStore', function() {
await store.write(hash, block);
}
const first = await fs.stat(store.filepath(0));
const second = await fs.stat(store.filepath(1));
const third = await fs.stat(store.filepath(2));
const first = await fs.stat(store.filepath(types.BLOCK, 0));
const second = await fs.stat(store.filepath(types.BLOCK, 1));
const third = await fs.stat(store.filepath(types.BLOCK, 2));
const len = first.size + second.size + third.size - (8 * 16);
assert.equal(len, 128 * 16);
@ -543,16 +603,50 @@ describe('BlockStore', function() {
assert.strictEqual(pruned, true);
}
assert.equal(await fs.exists(store.filepath(0)), false);
assert.equal(await fs.exists(store.filepath(1)), false);
assert.equal(await fs.exists(store.filepath(2)), false);
assert.equal(await fs.exists(store.filepath(types.BLOCK, 0)), false);
assert.equal(await fs.exists(store.filepath(types.BLOCK, 1)), false);
assert.equal(await fs.exists(store.filepath(types.BLOCK, 2)), false);
for (let i = 0; i < 16; i++) {
const exists = await store.has(hashes[i]);
assert.strictEqual(exists, false);
}
const exists = await store.db.has(layout.f.encode(0));
const exists = await store.db.has(layout.f.encode(types.BLOCK, 0));
assert.strictEqual(exists, false);
});
it('will prune block undo coins', async () => {
const hashes = [];
for (let i = 0; i < 16; i++) {
const block = random.randomBytes(128);
const hash = random.randomBytes(32);
hashes.push(hash);
await store.writeUndo(hash, block);
}
const first = await fs.stat(store.filepath(types.UNDO, 0));
const second = await fs.stat(store.filepath(types.UNDO, 1));
const third = await fs.stat(store.filepath(types.UNDO, 2));
const len = first.size + second.size + third.size - (8 * 16);
assert.equal(len, 128 * 16);
for (let i = 0; i < 16; i++) {
const pruned = await store.pruneUndo(hashes[i]);
assert.strictEqual(pruned, true);
}
assert.equal(await fs.exists(store.filepath(types.UNDO, 0)), false);
assert.equal(await fs.exists(store.filepath(types.UNDO, 1)), false);
assert.equal(await fs.exists(store.filepath(types.UNDO, 2)), false);
for (let i = 0; i < 16; i++) {
const exists = await store.hasUndo(hashes[i]);
assert.strictEqual(exists, false);
}
const exists = await store.db.has(layout.f.encode(types.UNDO, 0));
assert.strictEqual(exists, false);
});
});
@ -639,6 +733,17 @@ describe('BlockStore', function() {
assert.bufferEqual(block1, block2);
});
it('will write and read block undo coins', async () => {
const block1 = random.randomBytes(128);
const hash = random.randomBytes(32);
await store.writeUndo(hash, block1);
const block2 = await store.readUndo(hash);
assert.bufferEqual(block1, block2);
});
it('will read a block w/ offset and length', async () => {
const block1 = random.randomBytes(128);
const hash = random.randomBytes(32);
@ -687,6 +792,20 @@ describe('BlockStore', function() {
assert.strictEqual(exists, true);
});
it('will check if block undo coins exists (false)', async () => {
const hash = random.randomBytes(32);
const exists = await store.has(hash);
assert.strictEqual(exists, false);
});
it('will check if block undo coins exists (true)', async () => {
const block = random.randomBytes(128);
const hash = random.randomBytes(32);
await store.writeUndo(hash, block);
const exists = await store.hasUndo(hash);
assert.strictEqual(exists, true);
});
it('will prune blocks (true)', async () => {
const block = random.randomBytes(128);
const hash = random.randomBytes(32);
@ -704,5 +823,23 @@ describe('BlockStore', function() {
const pruned = await store.prune(hash);
assert.strictEqual(pruned, false);
});
it('will prune block undo coins (true)', async () => {
const block = random.randomBytes(128);
const hash = random.randomBytes(32);
await store.writeUndo(hash, block);
const pruned = await store.pruneUndo(hash);
assert.strictEqual(pruned, true);
const block2 = await store.readUndo(hash);
assert.strictEqual(block2, null);
});
it('will prune block undo coins (false)', async () => {
const hash = random.randomBytes(32);
const exists = await store.hasUndo(hash);
assert.strictEqual(exists, false);
const pruned = await store.pruneUndo(hash);
assert.strictEqual(pruned, false);
});
});
});