709 lines
19 KiB
JavaScript
709 lines
19 KiB
JavaScript
/* eslint-env mocha */
|
|
/* eslint prefer-arrow-callback: "off" */
|
|
|
|
'use strict';
|
|
|
|
const Logger = require('blgr');
|
|
const bio = require('bufio');
|
|
const assert = require('./util/assert');
|
|
const common = require('./util/common');
|
|
const {resolve} = require('path');
|
|
const fs = require('bfile');
|
|
const {rimraf} = require('./util/common');
|
|
const {mkdirp} = require('bfile');
|
|
const random = require('bcrypto/lib/random');
|
|
|
|
const vectors = [
|
|
common.readBlock('block300025'),
|
|
common.readBlock('block426884'),
|
|
common.readBlock('block898352')
|
|
];
|
|
|
|
const {
|
|
AbstractBlockStore,
|
|
FileBlockStore,
|
|
LevelBlockStore
|
|
} = require('../lib/blockstore');
|
|
|
|
const layout = require('../lib/blockstore/layout');
|
|
|
|
const {
|
|
BlockRecord,
|
|
FileRecord
|
|
} = require('../lib/blockstore/records');
|
|
|
|
describe('BlockStore', function() {
|
|
describe('Abstract', function() {
|
|
let logger = null;
|
|
|
|
function context(ctx) {
|
|
return {info: () => ctx};
|
|
}
|
|
|
|
beforeEach(() => {
|
|
logger = Logger.global;
|
|
Logger.global = {context};
|
|
});
|
|
|
|
afterEach(() => {
|
|
Logger.global = logger;
|
|
});
|
|
|
|
it('construct with custom logger', async () => {
|
|
const store = new AbstractBlockStore({logger: {context}});
|
|
assert(store.logger);
|
|
assert(store.logger.info);
|
|
assert.equal(store.logger.info(), 'blockstore');
|
|
});
|
|
|
|
it('construct with default logger', async () => {
|
|
const store = new AbstractBlockStore();
|
|
assert(store.logger);
|
|
assert(store.logger.info);
|
|
assert.equal(store.logger.info(), 'blockstore');
|
|
});
|
|
|
|
it('has unimplemented base methods', async () => {
|
|
const methods = ['open', 'close', 'write', 'read',
|
|
'prune', 'has'];
|
|
|
|
const store = new AbstractBlockStore();
|
|
|
|
for (const method of methods) {
|
|
assert(store[method]);
|
|
|
|
let err = null;
|
|
try {
|
|
await store[method]();
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
assert(err, `Expected unimplemented method ${method}.`);
|
|
assert.equal(err.message, 'Abstract method.');
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('Records', function() {
|
|
describe('BlockRecord', function() {
|
|
function constructError(options) {
|
|
let err = null;
|
|
|
|
try {
|
|
new BlockRecord({
|
|
file: options.file,
|
|
position: options.position,
|
|
length: options.length
|
|
});
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
|
|
assert(err);
|
|
}
|
|
|
|
function toAndFromRaw(options) {
|
|
const rec1 = new BlockRecord(options);
|
|
assert.equal(rec1.file, options.file);
|
|
assert.equal(rec1.position, options.position);
|
|
assert.equal(rec1.length, options.length);
|
|
|
|
const raw = rec1.toRaw();
|
|
const rec2 = BlockRecord.fromRaw(raw);
|
|
assert.equal(rec2.file, options.file);
|
|
assert.equal(rec2.position, options.position);
|
|
assert.equal(rec2.length, options.length);
|
|
}
|
|
|
|
it('construct with correct options', () => {
|
|
const rec = new BlockRecord({
|
|
file: 12,
|
|
position: 23392,
|
|
length: 4194304
|
|
});
|
|
assert.equal(rec.file, 12);
|
|
assert.equal(rec.position, 23392);
|
|
assert.equal(rec.length, 4194304);
|
|
});
|
|
|
|
it('construct null record', () => {
|
|
const rec = new BlockRecord();
|
|
assert.equal(rec.file, 0);
|
|
assert.equal(rec.position, 0);
|
|
assert.equal(rec.length, 0);
|
|
});
|
|
|
|
it('fail with signed number (file)', () => {
|
|
constructError({file: -1, position: 1, length: 1});
|
|
});
|
|
|
|
it('fail with signed number (position)', () => {
|
|
constructError({file: 1, position: -1, length: 1});
|
|
});
|
|
|
|
it('fail with signed number (length)', () => {
|
|
constructError({file: 1, position: 1, length: -1});
|
|
});
|
|
|
|
it('fail with non-32-bit number (file)', () => {
|
|
constructError({file: Math.pow(2, 32), position: 1, length: 1});
|
|
});
|
|
|
|
it('fail with non-32-bit number (position)', () => {
|
|
constructError({file: 1, position: Math.pow(2, 32), length: 1});
|
|
});
|
|
|
|
it('fail with non-32-bit number (length)', () => {
|
|
constructError({file: 1, position: 1, length: Math.pow(2, 32)});
|
|
});
|
|
|
|
it('construct with max 32-bit numbers', () => {
|
|
const max = Math.pow(2, 32) - 1;
|
|
|
|
const rec = new BlockRecord({
|
|
file: max,
|
|
position: max,
|
|
length: max
|
|
});
|
|
|
|
assert(rec);
|
|
assert.equal(rec.file, max);
|
|
assert.equal(rec.position, max);
|
|
assert.equal(rec.length, max);
|
|
});
|
|
|
|
it('serialize/deserialize file record (min)', () => {
|
|
toAndFromRaw({file: 0, position: 0, length: 0});
|
|
});
|
|
|
|
it('serialize/deserialize file record', () => {
|
|
toAndFromRaw({file: 12, position: 23392, length: 4194304});
|
|
});
|
|
|
|
it('serialize/deserialize file record (max)', () => {
|
|
const max = Math.pow(2, 32) - 1;
|
|
toAndFromRaw({file: max, position: max, length: max});
|
|
});
|
|
});
|
|
|
|
describe('FileRecord', function() {
|
|
function constructError(options) {
|
|
let err = null;
|
|
|
|
try {
|
|
new FileRecord({
|
|
blocks: options.blocks,
|
|
used: options.used,
|
|
length: options.length
|
|
});
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
|
|
assert(err);
|
|
}
|
|
|
|
function toAndFromRaw(options) {
|
|
const rec1 = new FileRecord(options);
|
|
assert.equal(rec1.blocks, options.blocks);
|
|
assert.equal(rec1.used, options.used);
|
|
assert.equal(rec1.length, options.length);
|
|
|
|
const raw = rec1.toRaw();
|
|
const rec2 = FileRecord.fromRaw(raw);
|
|
assert.equal(rec2.blocks, options.blocks);
|
|
assert.equal(rec2.used, options.used);
|
|
assert.equal(rec2.length, options.length);
|
|
}
|
|
|
|
it('construct with correct options', () => {
|
|
const rec = new FileRecord({
|
|
blocks: 1,
|
|
used: 4194304,
|
|
length: 20971520
|
|
});
|
|
assert.equal(rec.blocks, 1);
|
|
assert.equal(rec.used, 4194304);
|
|
assert.equal(rec.length, 20971520);
|
|
});
|
|
|
|
it('fail to with signed number (blocks)', () => {
|
|
constructError({blocks: -1, used: 1, length: 1});
|
|
});
|
|
|
|
it('fail to with signed number (used)', () => {
|
|
constructError({blocks: 1, used: -1, length: 1});
|
|
});
|
|
|
|
it('fail to with signed number (length)', () => {
|
|
constructError({blocks: 1, used: 1, length: -1});
|
|
});
|
|
|
|
it('fail to with non-32-bit number (blocks)', () => {
|
|
constructError({blocks: Math.pow(2, 32), used: 1, length: 1});
|
|
});
|
|
|
|
it('fail to with non-32-bit number (used)', () => {
|
|
constructError({blocks: 1, used: Math.pow(2, 32), length: 1});
|
|
});
|
|
|
|
it('fail to with non-32-bit number (length)', () => {
|
|
constructError({blocks: 1, used: 1, length: Math.pow(2, 32)});
|
|
});
|
|
|
|
it('serialize/deserialize block record (min)', () => {
|
|
toAndFromRaw({blocks: 0, used: 0, length: 0});
|
|
});
|
|
|
|
it('serialize/deserialize block record', () => {
|
|
toAndFromRaw({blocks: 10, used: 4194304, length: 20971520});
|
|
});
|
|
|
|
it('serialize/deserialize block record (max)', () => {
|
|
const max = Math.pow(2, 32) - 1;
|
|
toAndFromRaw({blocks: max, used: max, length: max});
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('FileBlockStore (Unit)', function() {
|
|
const location = '/tmp/.bcoin/blocks';
|
|
let store = null;
|
|
|
|
before(() => {
|
|
store = new FileBlockStore({
|
|
location: location,
|
|
maxFileLength: 1024
|
|
});
|
|
});
|
|
|
|
describe('allocate', function() {
|
|
it('will fail with length above file max', async () => {
|
|
let err = null;
|
|
try {
|
|
await store.allocate(1025);
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
assert(err);
|
|
assert.equal(err.message, 'Block length above max file length.');
|
|
});
|
|
});
|
|
|
|
describe('filepath', function() {
|
|
it('will give correct path (0)', () => {
|
|
const filepath = store.filepath(0);
|
|
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00000.dat');
|
|
});
|
|
|
|
it('will give correct path (1)', () => {
|
|
const filepath = store.filepath(7);
|
|
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00007.dat');
|
|
});
|
|
|
|
it('will give correct path (2)', () => {
|
|
const filepath = store.filepath(23);
|
|
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00023.dat');
|
|
});
|
|
|
|
it('will give correct path (3)', () => {
|
|
const filepath = store.filepath(456);
|
|
assert.equal(filepath, '/tmp/.bcoin/blocks/blk00456.dat');
|
|
});
|
|
|
|
it('will give correct path (4)', () => {
|
|
const filepath = store.filepath(8999);
|
|
assert.equal(filepath, '/tmp/.bcoin/blocks/blk08999.dat');
|
|
});
|
|
|
|
it('will give correct path (5)', () => {
|
|
const filepath = store.filepath(99999);
|
|
assert.equal(filepath, '/tmp/.bcoin/blocks/blk99999.dat');
|
|
});
|
|
|
|
it('will fail over max size', () => {
|
|
let err = null;
|
|
try {
|
|
store.filepath(100000);
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
|
|
assert(err);
|
|
assert.equal(err.message, 'File number too large.');
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('FileBlockStore (Integration 1)', function() {
|
|
const location = '/tmp/bcoin-blockstore-test';
|
|
let store = null;
|
|
|
|
beforeEach(async () => {
|
|
await rimraf(location);
|
|
await mkdirp(location);
|
|
|
|
store = new FileBlockStore({
|
|
location: location,
|
|
maxFileLength: 1024
|
|
});
|
|
|
|
await store.open();
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await store.close();
|
|
});
|
|
|
|
it('will write and read a block', async () => {
|
|
const block1 = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
await store.write(hash, block1);
|
|
|
|
const block2 = await store.read(hash);
|
|
|
|
assert.bufferEqual(block1, block2);
|
|
});
|
|
|
|
it('will read a block w/ offset and length', async () => {
|
|
const block1 = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
await store.write(hash, block1);
|
|
|
|
const offset = 79;
|
|
const size = 15;
|
|
|
|
const block2 = await store.read(hash, offset, size);
|
|
|
|
assert.bufferEqual(block1.slice(offset, offset + size), block2);
|
|
});
|
|
|
|
it('will fail to read w/ out-of-bounds length', async () => {
|
|
const block1 = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
await store.write(hash, block1);
|
|
|
|
const offset = 79;
|
|
const size = 50;
|
|
|
|
let err = null;
|
|
try {
|
|
await store.read(hash, offset, size);
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
|
|
assert(err);
|
|
assert.equal(err.message, 'Out-of-bounds read.');
|
|
});
|
|
|
|
it('will allocate new files', async () => {
|
|
const blocks = [];
|
|
|
|
for (let i = 0; i < 16; i++) {
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
blocks.push({hash, block});
|
|
await store.write(hash, block);
|
|
const block2 = await store.read(hash);
|
|
assert.bufferEqual(block2, block);
|
|
}
|
|
|
|
const first = await fs.stat(store.filepath(0));
|
|
const second = await fs.stat(store.filepath(1));
|
|
const third = await fs.stat(store.filepath(2));
|
|
assert.equal(first.size, 952);
|
|
assert.equal(second.size, 952);
|
|
assert.equal(third.size, 272);
|
|
|
|
const len = first.size + second.size + third.size - (8 * 16);
|
|
assert.equal(len, 128 * 16);
|
|
|
|
for (let i = 0; i < 16; i++) {
|
|
const expect = blocks[i];
|
|
const block = await store.read(expect.hash);
|
|
assert.bufferEqual(block, expect.block);
|
|
}
|
|
});
|
|
|
|
it('will recover from interrupt during block write', async () => {
|
|
{
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
await store.write(hash, block);
|
|
|
|
const block2 = await store.read(hash);
|
|
assert.bufferEqual(block2, block);
|
|
}
|
|
|
|
// Manually insert a partially written block to the
|
|
// end of file as would be the case of an untimely
|
|
// interrupted write of a block. The file record
|
|
// would not be updated to include the used bytes and
|
|
// thus this data should be overwritten.
|
|
{
|
|
const filepath = store.filepath(0);
|
|
|
|
const fd = await fs.open(filepath, 'a');
|
|
|
|
const bw = bio.write(8);
|
|
bw.writeU32(store.network.magic);
|
|
bw.writeU32(73);
|
|
const magic = bw.render();
|
|
|
|
const failblock = random.randomBytes(73);
|
|
|
|
const mwritten = await fs.write(fd, magic, 0, 8);
|
|
const bwritten = await fs.write(fd, failblock, 0, 73);
|
|
|
|
await fs.close(fd);
|
|
|
|
assert.equal(mwritten, 8);
|
|
assert.equal(bwritten, 73);
|
|
}
|
|
|
|
// Now check that this block has the correct position
|
|
// in the file and that it can be read correctly.
|
|
{
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
await store.write(hash, block);
|
|
|
|
const block2 = await store.read(hash);
|
|
assert.bufferEqual(block2, block);
|
|
}
|
|
});
|
|
|
|
it('will not write blocks at the same position', (done) => {
|
|
let err = null;
|
|
let finished = 0;
|
|
|
|
for (let i = 0; i < 16; i++) {
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
// Accidently don't use `await` and attempt to
|
|
// write multiple blocks in parallel and at the
|
|
// same file position.
|
|
const promise = store.write(hash, block);
|
|
promise.catch((e) => {
|
|
err = e;
|
|
}).finally(() => {
|
|
finished += 1;
|
|
if (finished >= 16) {
|
|
assert(err);
|
|
assert(err.message, 'Already writing.');
|
|
done();
|
|
}
|
|
});
|
|
}
|
|
});
|
|
|
|
it('will return null if block not found', async () => {
|
|
const hash = random.randomBytes(32);
|
|
const block = await store.read(hash);
|
|
assert.strictEqual(block, null);
|
|
});
|
|
|
|
it('will check if block exists (false)', async () => {
|
|
const hash = random.randomBytes(32);
|
|
const exists = await store.has(hash);
|
|
assert.strictEqual(exists, false);
|
|
});
|
|
|
|
it('will check if block exists (true)', async () => {
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
await store.write(hash, block);
|
|
const exists = await store.has(hash);
|
|
assert.strictEqual(exists, true);
|
|
});
|
|
|
|
it('will prune blocks', async () => {
|
|
const hashes = [];
|
|
for (let i = 0; i < 16; i++) {
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
hashes.push(hash);
|
|
await store.write(hash, block);
|
|
}
|
|
|
|
const first = await fs.stat(store.filepath(0));
|
|
const second = await fs.stat(store.filepath(1));
|
|
const third = await fs.stat(store.filepath(2));
|
|
|
|
const len = first.size + second.size + third.size - (8 * 16);
|
|
assert.equal(len, 128 * 16);
|
|
|
|
for (let i = 0; i < 16; i++) {
|
|
const pruned = await store.prune(hashes[i]);
|
|
assert.strictEqual(pruned, true);
|
|
}
|
|
|
|
assert.equal(await fs.exists(store.filepath(0)), false);
|
|
assert.equal(await fs.exists(store.filepath(1)), false);
|
|
assert.equal(await fs.exists(store.filepath(2)), false);
|
|
|
|
for (let i = 0; i < 16; i++) {
|
|
const exists = await store.has(hashes[i]);
|
|
assert.strictEqual(exists, false);
|
|
}
|
|
|
|
const exists = await store.db.has(layout.f.encode(0));
|
|
assert.strictEqual(exists, false);
|
|
});
|
|
});
|
|
|
|
describe('FileBlockStore (Integration 2)', function() {
|
|
const location = '/tmp/bcoin-blockstore-test';
|
|
let store = null;
|
|
|
|
beforeEach(async () => {
|
|
await rimraf(location);
|
|
await mkdirp(location);
|
|
|
|
store = new FileBlockStore({
|
|
location: location,
|
|
maxFileLength: 1024 * 1024
|
|
});
|
|
|
|
await store.open();
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await store.close();
|
|
});
|
|
|
|
it('will import from files (e.g. db corruption)', async () => {
|
|
const blocks = [];
|
|
|
|
for (let i = 0; i < vectors.length; i++) {
|
|
const [block] = vectors[i].getBlock();
|
|
const hash = block.hash();
|
|
const raw = block.toRaw();
|
|
|
|
blocks.push({hash, block: raw});
|
|
await store.write(hash, raw);
|
|
}
|
|
|
|
await store.close();
|
|
|
|
await rimraf(resolve(location, './index'));
|
|
|
|
store = new FileBlockStore({
|
|
location: location,
|
|
maxFileLength: 1024
|
|
});
|
|
|
|
await store.open();
|
|
|
|
for (let i = 0; i < vectors.length; i++) {
|
|
const expect = blocks[i];
|
|
const block = await store.read(expect.hash);
|
|
assert.equal(block.length, expect.block.length);
|
|
assert.bufferEqual(block, expect.block);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('LevelBlockStore', function() {
|
|
const location = '/tmp/bcoin-blockstore-test';
|
|
let store = null;
|
|
|
|
beforeEach(async () => {
|
|
await rimraf(location);
|
|
await mkdirp(location);
|
|
|
|
store = new LevelBlockStore({
|
|
location: location
|
|
});
|
|
|
|
await store.open();
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await store.close();
|
|
});
|
|
|
|
it('will write and read a block', async () => {
|
|
const block1 = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
await store.write(hash, block1);
|
|
|
|
const block2 = await store.read(hash);
|
|
|
|
assert.bufferEqual(block1, block2);
|
|
});
|
|
|
|
it('will read a block w/ offset and length', async () => {
|
|
const block1 = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
await store.write(hash, block1);
|
|
|
|
const offset = 79;
|
|
const size = 15;
|
|
|
|
const block2 = await store.read(hash, offset, size);
|
|
|
|
assert.bufferEqual(block1.slice(offset, offset + size), block2);
|
|
});
|
|
|
|
it('will fail to read w/ out-of-bounds length', async () => {
|
|
const block1 = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
|
|
await store.write(hash, block1);
|
|
|
|
const offset = 79;
|
|
const size = 50;
|
|
|
|
let err = null;
|
|
try {
|
|
await store.read(hash, offset, size);
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
|
|
assert(err);
|
|
assert.equal(err.message, 'Out-of-bounds read.');
|
|
});
|
|
|
|
it('will check if block exists (false)', async () => {
|
|
const hash = random.randomBytes(32);
|
|
const exists = await store.has(hash);
|
|
assert.strictEqual(exists, false);
|
|
});
|
|
|
|
it('will check if block exists (true)', async () => {
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
await store.write(hash, block);
|
|
const exists = await store.has(hash);
|
|
assert.strictEqual(exists, true);
|
|
});
|
|
|
|
it('will prune blocks (true)', async () => {
|
|
const block = random.randomBytes(128);
|
|
const hash = random.randomBytes(32);
|
|
await store.write(hash, block);
|
|
const pruned = await store.prune(hash);
|
|
assert.strictEqual(pruned, true);
|
|
const block2 = await store.read(hash);
|
|
assert.strictEqual(block2, null);
|
|
});
|
|
|
|
it('will prune blocks (false)', async () => {
|
|
const hash = random.randomBytes(32);
|
|
const exists = await store.has(hash);
|
|
assert.strictEqual(exists, false);
|
|
const pruned = await store.prune(hash);
|
|
assert.strictEqual(pruned, false);
|
|
});
|
|
});
|
|
});
|