diff --git a/CHANGELOG.md b/CHANGELOG.md index 19cde791..6821a499 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,26 @@ # Bcoin release notes & changelog -## v2.0.0 +## v2.0.0-dev + +### How to upgrade + +The way that block data is stored has changed for greater performance, +efficiency, reliability and portability. To upgrade to the new disk layout +it's necessary to move block data from LevelDB (e.g. `~/.bcoin/chain`) to +a new file based block storage (e.g. `~./.bcoin/blocks`). + +To do this you can run: +``` +node ./migrate/chaindb4to5.js /path/to/bcoin/chain +``` + +The migration will take 1-3 hours, depending on hardware. The block data +will now be stored at `/path/to/bcoin/blocks`, after the data has been moved +the chain database will be compacted to free disk space. + +Alternatively, you can also sync the chain again, however the above +migration will be faster as additional network bandwidth won't be used +for downloading the blocks again. ### Wallet API changes @@ -79,6 +99,7 @@ ### Other changes +- A new module for storing block data in files. - Use of `buffer-map` for storing hashes (see https://github.com/bcoin-org/bcoin/issues/533). - Use of `bsert` for assertions. diff --git a/bench/blockstore.js b/bench/blockstore.js new file mode 100644 index 00000000..817633b0 --- /dev/null +++ b/bench/blockstore.js @@ -0,0 +1,476 @@ +/*! + * bench/blockstore.js - benchmark blockstore for bcoin + * + * This can be run to benchmark the performance of the blockstore + * module for writing, reading and pruning block data. Results are + * written to stdout as JSON or formated bench results. + * + * Usage: + * node ./blockstore.js [--maxfile=] [--total=] + * [--location=] [--store=] + * [--output=] [--unsafe] + * + * Options: + * - `maxfile` The maximum file size (applies to "file" store). + * - `total` The total number of block bytes to write. + * - `location` The location to store block data. + * - `store` This can be "file" or "level". + * - `output` This can be "json", "bench" or "benchjson". + * - `unsafe` This will allocate block data directly from memory + * instead of random, it is faster. + * + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +process.title = 'blockstore-bench'; + +const {isAbsolute} = require('path'); +const {mkdirp} = require('bfile'); +const random = require('bcrypto/lib/random'); +const {BufferMap} = require('buffer-map'); + +const { + FileBlockStore, + LevelBlockStore +} = require('../lib/blockstore'); + +const config = { + 'maxfile': { + value: true, + parse: a => parseInt(a), + valid: a => Number.isSafeInteger(a), + fallback: 128 * 1024 * 1024 + }, + 'total': { + value: true, + parse: a => parseInt(a), + valid: a => Number.isSafeInteger(a), + fallback: 3 * 1024 * 1024 * 1024 + }, + 'location': { + value: true, + valid: a => isAbsolute(a), + fallback: '/tmp/bcoin-bench-blockstore' + }, + 'store': { + value: true, + valid: a => (a === 'file' || a === 'level'), + fallback: 'file' + }, + 'output': { + value: true, + valid: a => (a === 'json' || a === 'bench' || a === 'benchjson'), + fallback: 'bench' + }, + 'unsafe': { + value: false, + valid: a => (a === true || a === false), + fallback: false + } +}; + +/** + * These block sizes were generated from bitcoin mainnet blocks by putting + * sizes into bins of 256 ^ (2 * n) as the upper bound and calculating + * the percentage of each and then distributing to roughly match the + * percentage of the following: + * + * |-------------|------------| + * | percentage | bytes | + * |-------------|------------| + * | 23.4055 | 1048576 | + * | 15.5338 | 256 | + * | 12.2182 | 262144 | + * | 8.4079 | 524288 | + * | 7.1289 | 131072 | + * | 6.9197 | 65536 | + * | 6.7073 | 2097152 | + * | 4.6753 | 32768 | + * | 3.9695 | 4096 | + * | 3.3885 | 16384 | + * | 2.6526 | 8192 | + * | 2.0048 | 512 | + * | 1.587 | 1024 | + * | 1.3976 | 2048 | + * | 0.0032 | 4194304 | + * |-------------|------------| + */ + +const distribution = [ + 1048576, 256, 256, 524288, 262144, 256, 131072, 256, 524288, 256, 131072, + 1048576, 262144, 1048576, 2097152, 256, 1048576, 65536, 256, 262144, 8192, + 32768, 32768, 256, 1048576, 524288, 2097152, 1024, 1048576, 1048576, 131072, + 131072, 262144, 512, 1048576, 1048576, 1024, 1048576, 1048576, 262144, 2048, + 262144, 256, 1048576, 131072, 4096, 524288, 65536, 4096, 65536, 131072, + 2097152, 2097152, 2097152, 256, 524288, 4096, 262144, 65536, 65536, 262144, + 16384, 1048576, 32768, 262144, 1048576, 256, 131072, 1048576, 1048576, + 1048576, 8192, 1048576, 256, 16384, 1048576, 256, 256, 524288, 256, 32768, + 16384, 32768, 1048576, 512, 4096, 1048576, 1048576, 524288, 65536, 2097152, + 512, 262144, 8192, 524288, 131072, 65536, 16384, 2048, 262144, 1048576, + 1048576, 256, 524288, 262144, 4194304, 262144, 2097152 +]; + +(async () => { + let settings = null; + try { + settings = processArgs(process.argv, config); + } catch (err) { + console.log(err.message); + process.exit(1); + } + + await mkdirp(settings.location); + + let store = null; + let output = null; + + if (settings.store === 'file') { + store = new FileBlockStore({ + location: settings.location, + maxFileLength: settings.maxfile + }); + } else if (settings.store === 'level') { + store = new LevelBlockStore({ + location: settings.location + }); + } + + if (settings.output === 'bench') { + output = new BenchOutput(); + } else if (settings.output === 'benchjson') { + output = new BenchJSONOutput(); + } else if (settings.output === 'json') { + output = new JSONOutput(); + } + + await store.open(); + + const hashes = []; + const lengths = new BufferMap(); + + output.start(); + + // 1. Write data to the block store + let written = 0; + + async function write() { + for (const length of distribution) { + const hash = random.randomBytes(32); + let raw = null; + if (settings.unsafe) { + raw = Buffer.allocUnsafe(length); + } else { + raw = random.randomBytes(length); + } + + const start = process.hrtime(); + await store.write(hash, raw); + const elapsed = process.hrtime(start); + + hashes.push(hash); + lengths.set(hash, length); + written += length; + + output.result('write', start, elapsed, length); + + if (written >= settings.total) + break; + } + } + + while (written < settings.total) + await write(); + + // 2. Read data from the block store + for (const hash of hashes) { + const start = process.hrtime(); + const raw = await store.read(hash); + const elapsed = process.hrtime(start); + + output.result('read', start, elapsed, raw.length); + } + + // 3. Read data not in the order it was written (random) + for (let i = 0; i < hashes.length; i++) { + const rand = random.randomInt() / 0xffffffff * (hashes.length - 1) | 0; + const hash = hashes[rand]; + + const start = process.hrtime(); + const raw = await store.read(hash); + const elapsed = process.hrtime(start); + + output.result('randomread', start, elapsed, raw.length); + } + + // 4. Prune data from the block store + for (const hash of hashes) { + const start = process.hrtime(); + await store.prune(hash); + const elapsed = process.hrtime(start); + const length = lengths.get(hash); + + output.result('prune', start, elapsed, length); + } + + output.end(); + + await store.close(); +})().catch((err) => { + console.error(err); + process.exit(1); +}); + +class JSONOutput { + constructor() { + this.time = process.hrtime(); + this.index = 0; + } + + start() { + process.stdout.write('['); + } + + result(type, start, elapsed, length) { + if (this.index > 0) + process.stdout.write(','); + + const since = [start[0] - this.time[0], start[1] - this.time[1]]; + const smicro = hrToMicro(since); + const emicro = hrToMicro(elapsed); + + process.stdout.write(`{"type":"${type}","start":${smicro},`); + process.stdout.write(`"elapsed":${emicro},"length":${length},`); + process.stdout.write(`"index":${this.index}}`); + + this.index += 1; + } + + end() { + process.stdout.write(']'); + } +} + +class BenchOutput { + constructor() { + this.time = process.hrtime(); + this.index = 0; + this.results = {}; + this.interval = null; + this.stdout = process.stdout; + } + + start() { + this.stdout.write('Starting benchmark...\n'); + this.interval = setInterval(() => { + this.stdout.write(`Operation count=${this.index}\n`); + }, 5000); + } + + result(type, start, elapsed, length) { + const micro = hrToMicro(elapsed); + + if (!this.results[type]) + this.results[type] = {}; + + if (!this.results[type][length]) + this.results[type][length] = []; + + this.results[type][length].push(micro); + + this.index += 1; + } + + end() { + clearInterval(this.interval); + + this.stdout.write('Benchmark finished.\n'); + + function format(value) { + if (typeof value === 'number') + value = value.toFixed(2); + + if (typeof value !== 'string') + value = value.toString(); + + while (value.length < 15) + value = `${value} `; + + return value; + } + + function title(value) { + if (typeof value !== 'string') + value = value.toString(); + + while (value.length < 85) + value = ` ${value} `; + + if (value.length > 85) + value = value.slice(0, 85); + + return value; + } + + for (const type in this.results) { + this.stdout.write('\n'); + this.stdout.write(`${title(type)}\n`); + this.stdout.write(`${'='.repeat(85)}\n`); + this.stdout.write(`${format('length')}`); + this.stdout.write(`${format('operations')}`); + this.stdout.write(`${format('min')}`); + this.stdout.write(`${format('max')}`); + this.stdout.write(`${format('average')}`); + this.stdout.write(`${format('median')}`); + this.stdout.write('\n'); + this.stdout.write(`${'-'.repeat(85)}\n`); + + for (const length in this.results[type]) { + const cal = calculate(this.results[type][length]); + + this.stdout.write(`${format(length)}`); + this.stdout.write(`${format(cal.operations.toString())}`); + this.stdout.write(`${format(cal.min)}`); + this.stdout.write(`${format(cal.max)}`); + this.stdout.write(`${format(cal.average)}`); + this.stdout.write(`${format(cal.median)}`); + this.stdout.write('\n'); + } + this.stdout.write('\n'); + } + this.stdout.write('\n'); + } +} + +class BenchJSONOutput { + constructor() { + this.time = null; + this.results = {}; + this.stdout = process.stdout; + } + + start() { + this.time = process.hrtime(); + } + + result(type, start, elapsed, length) { + const micro = hrToMicro(elapsed); + + if (!this.results[type]) + this.results[type] = {}; + + if (!this.results[type][length]) + this.results[type][length] = []; + + this.results[type][length].push(micro); + } + + end() { + const report = { + summary: [], + time: hrToMicro(process.hrtime(this.time)), + elapsed: 0 + }; + + for (const type in this.results) { + for (const length in this.results[type]) { + const cal = calculate(this.results[type][length]); + + report.elapsed += cal.total; + + report.summary.push({ + type: type, + length: length, + operations: cal.operations, + min: cal.min, + max: cal.max, + average: cal.average, + median: cal.median + }); + } + } + + this.stdout.write(JSON.stringify(report, null, 2)); + this.stdout.write('\n'); + } +} + +function hrToMicro(time) { + return (time[0] * 1000000) + (time[1] / 1000); +} + +function calculate(times) { + times.sort((a, b) => a - b); + + let min = Infinity; + let max = 0; + + let total = 0; + + for (const micro of times) { + if (micro < min) + min = micro; + + if (micro > max) + max = micro; + + total += micro; + } + + const average = total / times.length; + const median = times[times.length / 2 | 0]; + const operations = times.length; + + return { + total, + operations, + min, + max, + average, + median + }; +} + +function processArgs(argv, config) { + const args = {}; + + for (const key in config) + args[key] = config[key].fallback; + + for (let i = 2; i < process.argv.length; i++) { + const arg = process.argv[i]; + const match = arg.match(/^(\-){1,2}([a-z]+)(\=)?(.*)?$/); + + if (!match) { + throw new Error(`Unexpected argument: ${arg}.`); + } else { + const key = match[2]; + let value = match[4]; + + if (!config[key]) + throw new Error(`Invalid argument: ${arg}.`); + + if (config[key].value && !value) { + value = process.argv[i + 1]; + i++; + } else if (!config[key].value && !value) { + value = true; + } else if (!config[key].value && value) { + throw new Error(`Unexpected value: ${key}=${value}`); + } + + if (config[key].parse) + value = config[key].parse(value); + + if (value) + args[key] = value; + + if (!config[key].valid(args[key])) + throw new Error(`Invalid value: ${key}=${value}`); + } + } + + return args; +} diff --git a/lib/bcoin.js b/lib/bcoin.js index b5e2a018..72ab240b 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -55,6 +55,7 @@ bcoin.set = function set(network) { // Blockchain bcoin.define('blockchain', './blockchain'); +bcoin.define('blockstore', './blockstore'); bcoin.define('Chain', './blockchain/chain'); bcoin.define('ChainEntry', './blockchain/chainentry'); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 5dab1d9f..7e94c6a7 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -49,6 +49,7 @@ class Chain extends AsyncEmitter { this.network = this.options.network; this.logger = this.options.logger.context('chain'); + this.blocks = this.options.blocks; this.workers = this.options.workers; this.db = new ChainDB(this.options); @@ -2662,6 +2663,7 @@ class ChainOptions { constructor(options) { this.network = Network.primary; this.logger = Logger.global; + this.blocks = null; this.workers = null; this.prefix = null; @@ -2695,6 +2697,13 @@ class ChainOptions { */ fromOptions(options) { + if (!options.spv) { + assert(options.blocks && typeof options.blocks === 'object', + 'Chain requires a blockstore.'); + } + + this.blocks = options.blocks; + if (options.network != null) this.network = Network.get(options.network); diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index a361df4b..f24e6f97 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -40,8 +40,10 @@ class ChainDB { this.options = options; this.network = this.options.network; this.logger = this.options.logger.context('chaindb'); + this.blocks = this.options.blocks; this.db = bdb.create(this.options); + this.stateCache = new StateCache(this.network); this.state = new ChainState(); this.pending = null; @@ -60,7 +62,7 @@ class ChainDB { this.logger.info('Opening ChainDB...'); await this.db.open(); - await this.db.verify(layout.V.encode(), 'chain', 4); + await this.db.verify(layout.V.encode(), 'chain', 5); const state = await this.getState(); @@ -101,7 +103,7 @@ class ChainDB { * @returns {Promise} */ - close() { + async close() { return this.db.close(); } @@ -760,7 +762,6 @@ class ChainDB { const start = pruneAfter + 1; const end = height - keepBlocks; - const b = this.db.batch(); for (let i = start; i <= end; i++) { const hash = await this.getHash(i); @@ -768,8 +769,8 @@ class ChainDB { if (!hash) throw new Error(`Cannot find hash for ${i}.`); - b.del(layout.b.encode(hash)); - b.del(layout.u.encode(hash)); + await this.blocks.pruneUndo(hash); + await this.blocks.prune(hash); } try { @@ -778,16 +779,12 @@ class ChainDB { const flags = ChainFlags.fromOptions(options); assert(flags.prune); - b.put(layout.O.encode(), flags.toRaw()); - - await b.write(); + await this.db.put(layout.O.encode(), flags.toRaw()); } catch (e) { options.prune = false; throw e; } - await this.db.compactRange(); - return true; } @@ -1014,7 +1011,7 @@ class ChainDB { */ async getUndoCoins(hash) { - const data = await this.db.get(layout.u.encode(hash)); + const data = await this.blocks.readUndo(hash); if (!data) return new UndoCoins(); @@ -1052,7 +1049,7 @@ class ChainDB { if (!hash) return null; - return this.db.get(layout.b.encode(hash)); + return this.blocks.read(hash); } /** @@ -1464,6 +1461,10 @@ class ChainDB { await this.commit(); + // Remove undo data _after_ successful commit. + if (this.blocks) + await this.blocks.pruneUndo(entry.hash); + return view; } @@ -1580,6 +1581,12 @@ class ChainDB { await this.commit(); + // Remove block and undo data _after_ successful commit. + if (this.blocks) { + await this.blocks.pruneUndo(tip.hash); + await this.blocks.prune(tip.hash); + } + // Update caches _after_ successful commit. this.cacheHeight.remove(tip.height); this.cacheHash.remove(tip.hash); @@ -1603,15 +1610,23 @@ class ChainDB { // one giant atomic write! this.start(); + let hashes = []; + try { for (const tip of tips) - await this._removeChain(tip); + hashes = hashes.concat(await this._removeChain(tip)); } catch (e) { this.drop(); throw e; } await this.commit(); + + // SPV doesn't store blocks. + if (this.blocks) { + for (const hash of hashes) + await this.blocks.prune(hash); + } } /** @@ -1629,6 +1644,8 @@ class ChainDB { this.logger.debug('Removing alternate chain: %h.', tip.hash); + const hashes = []; + for (;;) { if (await this.isMainChain(tip)) break; @@ -1639,7 +1656,10 @@ class ChainDB { this.del(layout.p.encode(tip.hash)); this.del(layout.h.encode(tip.hash)); this.del(layout.e.encode(tip.hash)); - this.del(layout.b.encode(tip.hash)); + + // Queue block to be pruned on + // successful write. + hashes.push(tip.hash); // Queue up hash to be removed // on successful write. @@ -1648,6 +1668,8 @@ class ChainDB { tip = await this.getPrevious(tip); assert(tip); } + + return hashes; } /** @@ -1665,9 +1687,8 @@ class ChainDB { if (this.options.spv) return; - // Write actual block data (this may be - // better suited to flat files in the future). - this.put(layout.b.encode(hash), block.toRaw()); + // Write actual block data. + await this.blocks.write(hash, block.toRaw()); if (!view) return; @@ -1691,8 +1712,6 @@ class ChainDB { if (!block) throw new Error('Block not found.'); - this.del(layout.b.encode(block.hash())); - return this.disconnectBlock(entry, block); } @@ -1762,7 +1781,7 @@ class ChainDB { // Write undo coins (if there are any). if (!view.undo.isEmpty()) - this.put(layout.u.encode(hash), view.undo.commit()); + await this.blocks.writeUndo(hash, view.undo.commit()); // Prune height-288 if pruning is enabled. return this.pruneBlock(entry); @@ -1820,9 +1839,6 @@ class ChainDB { // Commit new coin state. this.saveView(view); - // Remove undo coins. - this.del(layout.u.encode(hash)); - return view; } @@ -1851,8 +1867,8 @@ class ChainDB { if (!hash) return; - this.del(layout.b.encode(hash)); - this.del(layout.u.encode(hash)); + await this.blocks.pruneUndo(hash); + await this.blocks.prune(hash); } /** diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index 1bc801d5..01aaa086 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -19,10 +19,10 @@ const bdb = require('bdb'); * H[height] -> hash * n[hash] -> next hash * p[hash] -> tip index - * b[hash] -> block + * b[hash] -> block (deprecated) * t[hash] -> extended tx * c[hash] -> coins - * u[hash] -> undo coins + * u[hash] -> undo coins (deprecated) * v[bit][hash] -> versionbits state * T[addr-hash][hash] -> dummy (tx by address) * C[addr-hash][hash][index] -> dummy (coin by address) diff --git a/lib/blockstore/README.md b/lib/blockstore/README.md new file mode 100644 index 00000000..37fff012 --- /dev/null +++ b/lib/blockstore/README.md @@ -0,0 +1,147 @@ +# BlockStore + +BlockStore `lib/blockstore` is a bcoin module intended to be used as a backend +for storing block and undo coin data. It includes a backend that uses flat +files for storage. Its key benefit is performance improvements across the +board in disk I/O, which is the major bottleneck for the initial block sync. + +Blocks are stored in wire format directly to the disk, while some additional +metadata is stored in a key-value store, i.e. LevelDB, to help with the data +management. Both the flat files and the metadata db, are exposed through a +unified interace so that the users can simply read and write blocks without +having to worry about managing data layout on the disk. + +In addition to blocks, undo coin data, which is used to revert the changes +applied by a block (in case of a re-org), is also stored on disk, in a similar +fashion. + +## Interface + +The `AbstractBlockStore` interface defines the following abstract methods to be +defined by concrete implementations: + +### Basic housekeeping + +* `ensure()` +* `open()` +* `close()` + +### Block I/O + +* `read(hash, offset, size)` +* `write(hash, data)` +* `prune(hash)` +* `has(hash)` + +### Undo Coins I/O + +* `readUndo(hash)` +* `writeUndo(hash, data)` +* `pruneUndo(hash)` +* `hasUndo(hash)` + +The interface is implemented by `FileBlockStore` and `LevelBlockStore`, backed +by flat files and LevelDB respectively. We will focus here on the +`FileBlockStore`, which is the backend that implements a flat file based +storage. + +## FileBlockStore + +`FileBlockStore` implements the flat file backend for `AbstractBlockStore`. As +the name suggests, it uses flat files for block/undo data and LevelDB for +metadata. + +Let's create a file blockstore, write a block and walk-through the disk storage: + +```js +// nodejs +const store = blockstore.create({ + network: 'regtest', + prefix: '/tmp/blockstore' +}); +await store.ensure(); +await store.open(); +await store.write(hash, block); +``` + +```sh +// shell +tree /tmp/blockstore/ +/tmp/blockstore/ +└── blocks + ├── blk00000.dat + └── index + ├── LOG + ... +``` + +As we can see, the store writes to the file `blk00000.dat` in +`/tmp/blockstore/blocks/`, and the metadata is written to +`/tmp/blockstore/index`. + +Raw blocks are written to the disk in flat files named `blkXXXXX.dat`, where +`XXXXX` is the number of file being currently written, starting at +`blk00000.dat`. We store the file number as an integer in the metadata db, +expanding the digits to five places. + +The metadata db key `layout.F` tracks the last file used for writing. Each +file in turn tracks the number of blocks in it, the number of bytes used and +its max length. This data is stored in the db key `layout.f`. + + f['block'][0] => [1, 5, 128] // blk00000.dat: 1 block written, 5 bytes used, 128 bytes length + F['block'] => 0 // writing to file blk00000.dat + +Each raw block data is preceded by a magic marker defined as follows, to help +identify data written by us: + + magic (8 bytes) = network.magic (4 bytes) + block data length (4 bytes) + +For raw undo block data, the hash of the block is also included: + + magic (40 bytes) = network.magic (4 bytes) + length (4 bytes) + hash (32 bytes) + +But a marker alone is not sufficient to track the data we write to the files. +For each block we write, we need to store a pointer to the position in the file +where to start reading, and the size of the data we need to seek. This data is +stored in the metadata db using the key `layout.b`: + + b['block']['hash'] => [0, 8, 285] // 'hash' points to file blk00000.dat, position 8, size 285 + +Using this we know that our block is in `blk00000.dat`, bytes 8 through 285. + +Note that the position indicates that the block data is preceded by 8 bytes of +the magic marker. + + +Examples: + +> `store.write('hash', 'block')` + + blk00000: + 0xfabfb5da05000000 block + + index: + b['block']['hash'] => [0, 8, 5] + f['block'][0] => [1, 13, 128] + F['block'] => 0 + +> `store.write('hash1', 'block1')` + + blk00000: + 0xfabfb5da05000000 block 0xfabfb5da06000000 block1 + + index: + b['block']['hash'] => [0, 8, 5] + b['block']['hash1'] => [0, 13, 6] + f['block'][0] => [2, 19, 128] + F['block'] => 0 + +> `store.prune('hash1', 'block1')` + + blk00000: + 0xfabfb5da05000000 block 0xfabfb5da06000000 block1 + + index: + b['block']['hash'] => [0, 8, 5] + f['block'][0] => [1, 19, 128] + F['block'] => 0 diff --git a/lib/blockstore/abstract.js b/lib/blockstore/abstract.js new file mode 100644 index 00000000..efde8bd5 --- /dev/null +++ b/lib/blockstore/abstract.js @@ -0,0 +1,142 @@ +/*! + * blockstore/abstract.js - abstract blockstore for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const Logger = require('blgr'); + +/** + * Abstract Block Store + * + * @alias module:blockstore.AbstractBlockStore + * @abstract + */ + +class AbstractBlockStore { + /** + * Create an abstract blockstore. + * @constructor + */ + + constructor(options) { + this.options = options || {}; + + if (this.options.logger != null) + this.logger = this.options.logger.context('blockstore'); + else + this.logger = Logger.global.context('blockstore'); + } + + /** + * This method ensures that resources are available + * before opening. + * @returns {Promise} + */ + + async ensure() { + throw new Error('Abstract method.'); + } + + /** + * This method opens any necessary resources and + * initializes the store to be ready to be queried. + * @returns {Promise} + */ + + async open() { + throw new Error('Abstract method.'); + } + + /** + * This method closes resources and prepares + * the store to be closed. + * @returns {Promise} + */ + + async close() { + throw new Error('Abstract method.'); + } + + /** + * This method stores block undo coin data. + * @returns {Promise} + */ + + async writeUndo(hash, data) { + throw new Error('Abstract method.'); + } + + /** + * This method stores block data. + * @returns {Promise} + */ + + async write(hash, data) { + throw new Error('Abstract method.'); + } + + /** + * This method will retrieve block undo coin data. + * @returns {Promise} + */ + + async readUndo(hash) { + throw new Error('Abstract method.'); + } + + /** + * This method will retrieve block data. Smaller portions of + * the block can be read by using the offset and size arguments. + * @returns {Promise} + */ + + async read(hash, offset, size) { + throw new Error('Abstract method.'); + } + + /** + * This will free resources for storing the block undo coin data. + * @returns {Promise} + */ + + async pruneUndo(hash) { + throw new Error('Abstract method.'); + } + + /** + * This will free resources for storing the block data. + * @returns {Promise} + */ + + async prune(hash) { + throw new Error('Abstract method.'); + } + + /** + * This will check if a block undo coin data has been stored + * and is available. + * @returns {Promise} + */ + + async hasUndo(hash) { + throw new Error('Abstract method.'); + } + + /** + * This will check if a block has been stored and is available. + * @returns {Promise} + */ + + async has(hash) { + throw new Error('Abstract method.'); + } +} + +/* + * Expose + */ + +module.exports = AbstractBlockStore; diff --git a/lib/blockstore/common.js b/lib/blockstore/common.js new file mode 100644 index 00000000..f1cb310f --- /dev/null +++ b/lib/blockstore/common.js @@ -0,0 +1,31 @@ +/*! + * common.js - blockstore constants for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +/** + * @module blockstore/common + */ + +/** + * Block data types. + * @enum {Number} + */ + +exports.types = { + BLOCK: 1, + UNDO: 2 +}; + +/** + * File prefixes for block data types. + * @enum {String} + */ + +exports.prefixes = { + 1: 'blk', + 2: 'blu' +}; diff --git a/lib/blockstore/file.js b/lib/blockstore/file.js new file mode 100644 index 00000000..02733336 --- /dev/null +++ b/lib/blockstore/file.js @@ -0,0 +1,590 @@ +/*! + * blockstore/file.js - file blockstore for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const {isAbsolute, resolve, join} = require('path'); +const bdb = require('bdb'); +const assert = require('bsert'); +const fs = require('bfile'); +const bio = require('bufio'); +const hash256 = require('bcrypto/lib/hash256'); +const Network = require('../protocol/network'); +const AbstractBlockStore = require('./abstract'); +const {BlockRecord, FileRecord} = require('./records'); +const layout = require('./layout'); +const {types, prefixes} = require('./common'); + +/** + * File Block Store + * + * @alias module:blockstore:FileBlockStore + * @abstract + */ + +class FileBlockStore extends AbstractBlockStore { + /** + * Create a blockstore that stores blocks in files. + * @constructor + */ + + constructor(options) { + super(options); + + assert(isAbsolute(options.location), 'Location not absolute.'); + + this.location = options.location; + this.indexLocation = resolve(this.location, './index'); + + this.db = bdb.create({ + location: this.indexLocation, + cacheSize: options.cacheSize, + compression: false + }); + + this.maxFileLength = options.maxFileLength || 128 * 1024 * 1024; + + assert(Number.isSafeInteger(this.maxFileLength), + 'Invalid max file length.'); + + this.network = Network.primary; + + if (options.network != null) + this.network = Network.get(options.network); + + this.writing = false; + } + + /** + * Compares the number of files in the directory + * with the recorded number of files. + * @param {Number} type - The type of block data + * @private + * @returns {Promise} + */ + + async check(type) { + const prefix = prefixes[type]; + const regexp = new RegExp(`^${prefix}(\\d{5})\\.dat$`); + const all = await fs.readdir(this.location); + const dats = all.filter(f => regexp.test(f)); + const filenos = dats.map(f => parseInt(f.match(regexp)[1])); + + let missing = false; + + for (const fileno of filenos) { + const rec = await this.db.get(layout.f.encode(type, fileno)); + if (!rec) { + missing = true; + break; + } + } + + return {missing, filenos}; + } + + /** + * Creates indexes from files for a block type. Reads the hash of + * the block data from the magic prefix, except for a block which + * the hash is read from the block header. + * @private + * @param {Number} type - The type of block data + * @returns {Promise} + */ + + async _index(type) { + const {missing, filenos} = await this.check(type); + + if (!missing) + return; + + this.logger.info('Indexing block type %d...', type); + + for (const fileno of filenos) { + const b = this.db.batch(); + const filepath = this.filepath(type, fileno); + const data = await fs.readFile(filepath); + const reader = bio.read(data); + let magic = null; + let blocks = 0; + + while (reader.left() >= 4) { + magic = reader.readU32(); + + // Move forward a byte from the last read + // if the magic doesn't match. + if (magic !== this.network.magic) { + reader.seek(-3); + continue; + } + + let hash = null; + let position = 0; + let length = 0; + + try { + length = reader.readU32(); + + if (type === types.BLOCK) { + position = reader.offset; + hash = hash256.digest(reader.readBytes(80, true)); + reader.seek(length - 80); + } else { + hash = reader.readHash(); + position = reader.offset; + reader.seek(length); + } + } catch (err) { + this.logger.warning( + 'Unknown block in file: %s, reason: %s', + filepath, err.message); + continue; + } + + const blockrecord = new BlockRecord({ + file: fileno, + position: position, + length: length + }); + + blocks += 1; + b.put(layout.b.encode(type, hash), blockrecord.toRaw()); + } + + const filerecord = new FileRecord({ + blocks: blocks, + used: reader.offset, + length: this.maxFileLength + }); + + b.put(layout.f.encode(type, fileno), filerecord.toRaw()); + + await b.write(); + + this.logger.info('Indexed %d blocks (file=%s).', blocks, filepath); + } + } + + /** + * Compares the number of files in the directory + * with the recorded number of files. If there are any + * inconsistencies it will reindex all blocks. + * @private + * @returns {Promise} + */ + + async index() { + await this._index(types.BLOCK); + await this._index(types.UNDO); + } + + /** + * This method ensures that both the block storage directory + * and index directory exist. + * before opening. + * @returns {Promise} + */ + + async ensure() { + return fs.mkdirp(this.indexLocation); + } + + /** + * Opens the file block store. It will regenerate necessary block + * indexing if the index is missing or inconsistent. + * @returns {Promise} + */ + + async open() { + this.logger.info('Opening FileBlockStore...'); + + await this.db.open(); + await this.db.verify(layout.V.encode(), 'fileblockstore', 0); + + await this.index(); + } + + /** + * This closes the file block store and underlying + * indexing databases. + */ + + async close() { + this.logger.info('Closing FileBlockStore...'); + + await this.db.close(); + } + + /** + * This method will determine the file path based on the file number + * and the current block data location. + * @private + * @param {Number} type - The type of block data + * @param {Number} fileno - The number of the file. + * @returns {Promise} + */ + + filepath(type, fileno) { + const pad = 5; + + let num = fileno.toString(10); + + if (num.length > pad) + throw new Error('File number too large.'); + + while (num.length < pad) + num = `0${num}`; + + let filepath = null; + + const prefix = prefixes[type]; + + if (!prefix) + throw new Error('Unknown file prefix.'); + + filepath = join(this.location, `${prefix}${num}.dat`); + + return filepath; + } + + /** + * This method will select and potentially allocate a file to + * write a block based on the size and type. + * @private + * @param {Number} type - The type of block data + * @param {Number} length - The number of bytes + * @returns {Promise} + */ + + async allocate(type, length) { + if (length > this.maxFileLength) + throw new Error('Block length above max file length.'); + + let fileno = 0; + let filerecord = null; + let filepath = null; + + const last = await this.db.get(layout.F.encode(type)); + if (last) + fileno = bio.readU32(last, 0); + + filepath = this.filepath(type, fileno); + + const rec = await this.db.get(layout.f.encode(type, fileno)); + + let touch = false; + + if (rec) { + filerecord = FileRecord.fromRaw(rec); + } else { + touch = true; + filerecord = new FileRecord({ + blocks: 0, + used: 0, + length: this.maxFileLength + }); + } + + if (filerecord.used + length > filerecord.length) { + fileno += 1; + filepath = this.filepath(type, fileno); + touch = true; + filerecord = new FileRecord({ + blocks: 0, + used: 0, + length: this.maxFileLength + }); + } + + if (touch) { + const fd = await fs.open(filepath, 'w'); + await fs.close(fd); + } + + return {fileno, filerecord, filepath}; + } + + /** + * This method stores block undo coin data in files. + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async writeUndo(hash, data) { + return this._write(types.UNDO, hash, data); + } + + /** + * This method stores block data in files. + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async write(hash, data) { + return this._write(types.BLOCK, hash, data); + } + + /** + * This method stores block data in files with by appending + * data to the last written file and updating indexes to point + * to the file and position. + * @private + * @param {Number} type - The type of block data + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async _write(type, hash, data) { + if (this.writing) + throw new Error('Already writing.'); + + this.writing = true; + + if (await this.db.has(layout.b.encode(type, hash))) { + this.writing = false; + return false; + } + + let mlength = 8; + + // Hash for a block is not stored with + // the magic prefix as it's read from the header + // of the block data. + if (type !== types.BLOCK) + mlength += 32; + + const blength = data.length; + const length = data.length + mlength; + + const bwm = bio.write(mlength); + + bwm.writeU32(this.network.magic); + bwm.writeU32(blength); + + if (type !== types.BLOCK) + bwm.writeHash(hash); + + const magic = bwm.render(); + + const { + fileno, + filerecord, + filepath + } = await this.allocate(type, length); + + const mposition = filerecord.used; + const bposition = filerecord.used + mlength; + + const fd = await fs.open(filepath, 'r+'); + + let mwritten = 0; + let bwritten = 0; + + try { + mwritten = await fs.write(fd, magic, 0, mlength, mposition); + bwritten = await fs.write(fd, data, 0, blength, bposition); + } finally { + await fs.close(fd); + } + + if (mwritten !== mlength) { + this.writing = false; + throw new Error('Could not write block magic.'); + } + + if (bwritten !== blength) { + this.writing = false; + throw new Error('Could not write block.'); + } + + filerecord.blocks += 1; + filerecord.used += length; + + const b = this.db.batch(); + + const blockrecord = new BlockRecord({ + file: fileno, + position: bposition, + length: blength + }); + + b.put(layout.b.encode(type, hash), blockrecord.toRaw()); + b.put(layout.f.encode(type, fileno), filerecord.toRaw()); + + const last = bio.write(4).writeU32(fileno).render(); + b.put(layout.F.encode(type), last); + + await b.write(); + + this.writing = false; + + return true; + } + + /** + * This method will retrieve block undo coin data. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async readUndo(hash) { + return this._read(types.UNDO, hash); + } + + /** + * This method will retrieve block data. Smaller portions of the + * block (e.g. transactions) can be read by using the offset and + * length arguments. + * @param {Buffer} hash - The block hash + * @param {Number} offset - The offset within the block + * @param {Number} length - The number of bytes of the data + * @returns {Promise} + */ + + async read(hash, offset, length) { + return this._read(types.BLOCK, hash, offset, length); + } + + /** + * This methods reads data from disk by retrieving the index of + * the data and reading from the corresponding file and location. + * @private + * @param {Number} type - The type of block data + * @param {Buffer} hash - The block hash + * @param {Number} offset - The offset within the block + * @param {Number} length - The number of bytes of the data + * @returns {Promise} + */ + + async _read(type, hash, offset, length) { + const raw = await this.db.get(layout.b.encode(type, hash)); + if (!raw) + return null; + + const blockrecord = BlockRecord.fromRaw(raw); + + const filepath = this.filepath(type, blockrecord.file); + + let position = blockrecord.position; + + if (offset) + position += offset; + + if (!length) + length = blockrecord.length; + + if (offset + length > blockrecord.length) + throw new Error('Out-of-bounds read.'); + + const data = Buffer.alloc(length); + + const fd = await fs.open(filepath, 'r'); + let bytes = 0; + + try { + bytes = await fs.read(fd, data, 0, length, position); + } finally { + await fs.close(fd); + } + + if (bytes !== length) + throw new Error('Wrong number of bytes read.'); + + return data; + } + + /** + * This will free resources for storing the block undo coin data. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async pruneUndo(hash) { + return this._prune(types.UNDO, hash); + } + + /** + * This will free resources for storing the block data. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async prune(hash) { + return this._prune(types.BLOCK, hash); + } + + /** + * This will free resources for storing the block data. The block + * data may not be deleted from disk immediately, the index for the + * block is removed and will not be able to be read. The underlying + * file is unlinked when all blocks in a file have been pruned. + * @private + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async _prune(type, hash) { + const braw = await this.db.get(layout.b.encode(type, hash)); + if (!braw) + return false; + + const blockrecord = BlockRecord.fromRaw(braw); + + const fraw = await this.db.get(layout.f.encode(type, blockrecord.file)); + if (!fraw) + return false; + + const filerecord = FileRecord.fromRaw(fraw); + + filerecord.blocks -= 1; + + const b = this.db.batch(); + + if (filerecord.blocks === 0) + b.del(layout.f.encode(type, blockrecord.file)); + else + b.put(layout.f.encode(type, blockrecord.file), filerecord.toRaw()); + + b.del(layout.b.encode(type, hash)); + + await b.write(); + + if (filerecord.blocks === 0) + await fs.unlink(this.filepath(type, blockrecord.file)); + + return true; + } + + /** + * This will check if a block undo coin has been stored + * and is available. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async hasUndo(hash) { + return await this.db.has(layout.b.encode(types.UNDO, hash)); + } + + /** + * This will check if a block has been stored and is available. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async has(hash) { + return await this.db.has(layout.b.encode(types.BLOCK, hash)); + } +} + +/* + * Expose + */ + +module.exports = FileBlockStore; diff --git a/lib/blockstore/index.js b/lib/blockstore/index.js new file mode 100644 index 00000000..b8e4c634 --- /dev/null +++ b/lib/blockstore/index.js @@ -0,0 +1,42 @@ +/*! + * blockstore/index.js - bitcoin blockstore for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const {join} = require('path'); + +const AbstractBlockStore = require('./abstract'); +const LevelBlockStore = require('./level'); +const FileBlockStore = require('./file'); + +/** + * @module blockstore + */ + +exports.create = (options) => { + const location = join(options.prefix, 'blocks'); + + if (options.memory) { + return new LevelBlockStore({ + network: options.network, + logger: options.logger, + location: location, + cacheSize: options.cacheSize, + memory: options.memory + }); + } + + return new FileBlockStore({ + network: options.network, + logger: options.logger, + location: location, + cacheSize: options.cacheSize + }); +}; + +exports.AbstractBlockStore = AbstractBlockStore; +exports.FileBlockStore = FileBlockStore; +exports.LevelBlockStore = LevelBlockStore; diff --git a/lib/blockstore/layout.js b/lib/blockstore/layout.js new file mode 100644 index 00000000..ce5cf290 --- /dev/null +++ b/lib/blockstore/layout.js @@ -0,0 +1,30 @@ +/*! + * blockstore/layout.js - file blockstore data layout for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const bdb = require('bdb'); + +/* + * Database Layout: + * V -> db version + * F[type] -> last file record by type + * f[type][fileno] -> file record by type and file number + * b[type][hash] -> block record by type and block hash + */ + +const layout = { + V: bdb.key('V'), + F: bdb.key('F', ['uint32']), + f: bdb.key('f', ['uint32', 'uint32']), + b: bdb.key('b', ['uint32', 'hash256']) +}; + +/* + * Expose + */ + +module.exports = layout; diff --git a/lib/blockstore/level.js b/lib/blockstore/level.js new file mode 100644 index 00000000..bb0f7f6c --- /dev/null +++ b/lib/blockstore/level.js @@ -0,0 +1,189 @@ +/*! + * blockstore/level.js - leveldb blockstore for bcoin + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const bdb = require('bdb'); +const fs = require('bfile'); +const AbstractBlockStore = require('./abstract'); +const layout = require('./layout'); +const {types} = require('./common'); + +/** + * LevelDB Block Store + * + * @alias module:blockstore:LevelBlockStore + * @abstract + */ + +class LevelBlockStore extends AbstractBlockStore { + /** + * Create a blockstore that stores blocks in LevelDB. + * @constructor + */ + + constructor(options) { + super(options); + + this.location = options.location; + + this.db = bdb.create({ + location: this.location, + cacheSize: options.cacheSize, + compression: false, + memory: options.memory + }); + } + + /** + * This method ensures that the storage directory exists + * before opening. + * @returns {Promise} + */ + + async ensure() { + return fs.mkdirp(this.location); + } + + /** + * Opens the block storage. + * @returns {Promise} + */ + + async open() { + this.logger.info('Opening LevelBlockStore...'); + + await this.db.open(); + await this.db.verify(layout.V.encode(), 'levelblockstore', 0); + } + + /** + * Closes the block storage. + */ + + async close() { + this.logger.info('Closing LevelBlockStore...'); + + await this.db.close(); + } + + /** + * This method stores block undo coin data in LevelDB. + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async writeUndo(hash, data) { + return this.db.put(layout.b.encode(types.UNDO, hash), data); + } + + /** + * This method stores block data in LevelDB. + * @param {Buffer} hash - The block hash + * @param {Buffer} data - The block data + * @returns {Promise} + */ + + async write(hash, data) { + return this.db.put(layout.b.encode(types.BLOCK, hash), data); + } + + /** + * This method will retrieve block undo coin data. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async readUndo(hash) { + return this.db.get(layout.b.encode(types.UNDO, hash)); + } + + /** + * This method will retrieve block data. Smaller portions of the + * block (e.g. transactions) can be returned using the offset and + * length arguments. However, the entire block will be read as the + * data is stored in a key/value database. + * @param {Buffer} hash - The block hash + * @param {Number} offset - The offset within the block + * @param {Number} length - The number of bytes of the data + * @returns {Promise} + */ + + async read(hash, offset, length) { + let raw = await this.db.get(layout.b.encode(types.BLOCK, hash)); + + if (offset) { + if (offset + length > raw.length) + throw new Error('Out-of-bounds read.'); + + raw = raw.slice(offset, offset + length); + } + + return raw; + } + + /** + * This will free resources for storing the block undo coin data. + * The block data may not be immediately removed from disk, and will + * be reclaimed during LevelDB compaction. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async pruneUndo(hash) { + if (!await this.hasUndo(hash)) + return false; + + await this.db.del(layout.b.encode(types.UNDO, hash)); + + return true; + } + + /** + * This will free resources for storing the block data. The block + * data may not be immediately removed from disk, and will be reclaimed + * during LevelDB compaction. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async prune(hash) { + if (!await this.has(hash)) + return false; + + await this.db.del(layout.b.encode(types.BLOCK, hash)); + + return true; + } + + /** + * This will check if a block undo coin data has been stored + * and is available. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async hasUndo(hash) { + return this.db.has(layout.b.encode(types.UNDO, hash)); + } + + /** + * This will check if a block has been stored and is available. + * @param {Buffer} hash - The block hash + * @returns {Promise} + */ + + async has(hash) { + return this.db.has(layout.b.encode(types.BLOCK, hash)); + } +} + +/* + * Expose + */ + +module.exports = LevelBlockStore; diff --git a/lib/blockstore/records.js b/lib/blockstore/records.js new file mode 100644 index 00000000..d387171b --- /dev/null +++ b/lib/blockstore/records.js @@ -0,0 +1,149 @@ +/*! + * blockstore/records.js - blockstore records + * Copyright (c) 2019, Braydon Fuller (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +const assert = require('bsert'); +const bio = require('bufio'); + +/** + * @module blockstore/records + */ + +/** + * Block Record + */ + +class BlockRecord { + /** + * Create a block record. + * @constructor + */ + + constructor(options = {}) { + this.file = options.file || 0; + this.position = options.position || 0; + this.length = options.length || 0; + + assert((this.file >>> 0) === this.file); + assert((this.position >>> 0) === this.position); + assert((this.length >>> 0) === this.length); + } + + /** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.file = br.readU32(); + this.position = br.readU32(); + this.length = br.readU32(); + + return this; + } + + /** + * Instantiate block record from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {BlockRecord} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } + + /** + * Serialize the block record. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(12); + + bw.writeU32(this.file); + bw.writeU32(this.position); + bw.writeU32(this.length); + + return bw.render(); + } +} + +/** + * File Record + */ + +class FileRecord { + /** + * Create a file record. + * @constructor + */ + + constructor(options = {}) { + this.blocks = options.blocks || 0; + this.used = options.used || 0; + this.length = options.length || 0; + + assert((this.blocks >>> 0) === this.blocks); + assert((this.used >>> 0) === this.used); + assert((this.length >>> 0) === this.length); + } + + /** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + */ + + fromRaw(data) { + const br = bio.read(data); + + this.blocks = br.readU32(); + this.used = br.readU32(); + this.length = br.readU32(); + + return this; + } + + /** + * Instantiate file record from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {ChainState} + */ + + static fromRaw(data) { + return new this().fromRaw(data); + } + + /** + * Serialize the file record. + * @returns {Buffer} + */ + + toRaw() { + const bw = bio.write(12); + + bw.writeU32(this.blocks); + bw.writeU32(this.used); + bw.writeU32(this.length); + + return bw.render(); + } +} + +/* + * Expose + */ + +exports.BlockRecord = BlockRecord; +exports.FileRecord = FileRecord; + +module.exports = exports; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 0e6a8d0e..99e1ad9b 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -16,6 +16,7 @@ const Miner = require('../mining/miner'); const Node = require('./node'); const HTTP = require('./http'); const RPC = require('./rpc'); +const blockstore = require('../blockstore'); /** * Full Node @@ -40,10 +41,20 @@ class FullNode extends Node { // SPV flag. this.spv = false; - // Instantiate blockchain. + // Instantiate block storage. + this.blocks = blockstore.create({ + network: this.network, + logger: this.logger, + prefix: this.config.prefix, + cacheSize: this.config.mb('block-cache-size'), + memory: this.config.bool('memory') + }); + + // Chain needs access to blocks. this.chain = new Chain({ network: this.network, logger: this.logger, + blocks: this.blocks, workers: this.workers, memory: this.config.bool('memory'), prefix: this.config.prefix, @@ -218,6 +229,7 @@ class FullNode extends Node { this.opened = true; await this.handlePreopen(); + await this.blocks.open(); await this.chain.open(); await this.mempool.open(); await this.miner.open(); @@ -250,6 +262,7 @@ class FullNode extends Node { await this.miner.close(); await this.mempool.close(); await this.chain.close(); + await this.blocks.close(); await this.handleClose(); } diff --git a/lib/node/node.js b/lib/node/node.js index 099f4985..b407020e 100644 --- a/lib/node/node.js +++ b/lib/node/node.js @@ -57,6 +57,7 @@ class Node extends EventEmitter { this.workers = null; this.spv = false; + this.blocks = null; this.chain = null; this.fees = null; this.mempool = null; @@ -135,6 +136,9 @@ class Node extends EventEmitter { if (this.memory) return undefined; + if (this.blocks) + await this.blocks.ensure(); + return fs.mkdirp(this.config.prefix); } diff --git a/migrate/chaindb4to5.js b/migrate/chaindb4to5.js new file mode 100644 index 00000000..b9eb1f0b --- /dev/null +++ b/migrate/chaindb4to5.js @@ -0,0 +1,142 @@ +'use strict'; + +const assert = require('assert'); +const bdb = require('bdb'); +const layout = require('../lib/blockchain/layout'); +const FileBlockStore = require('../lib/blockstore/file'); +const {resolve} = require('path'); + +assert(process.argv.length > 2, 'Please pass in a database path.'); + +// migration - +// chaindb: leveldb to flat files + +const db = bdb.create({ + location: process.argv[2], + memory: false, + compression: true, + cacheSize: 32 << 20, + createIfMissing: false +}); + +const location = resolve(process.argv[2], '../blocks'); + +const blockStore = new FileBlockStore({ + location: location +}); + +async function updateVersion() { + const ver = await checkVersion(); + + console.log('Updating version to %d.', ver + 1); + + const buf = Buffer.allocUnsafe(5 + 4); + buf.write('chain', 0, 'ascii'); + buf.writeUInt32LE(5, 5, true); + + const parent = db.batch(); + parent.put(layout.V.encode(), buf); + await parent.write(); +} + +async function checkVersion() { + console.log('Checking version.'); + + const data = await db.get(layout.V.encode()); + assert(data, 'No version.'); + + const ver = data.readUInt32LE(5, true); + + if (ver !== 4) + throw Error(`DB is version ${ver}.`); + + return ver; +} + +async function migrateUndoBlocks() { + console.log('Migrating undo blocks'); + + let parent = db.batch(); + + const iter = db.iterator({ + gte: layout.u.min(), + lte: layout.u.max(), + keys: true, + values: true + }); + + let total = 0; + + await iter.each(async (key, value) => { + const hash = key.slice(1); + await blockStore.writeUndo(hash, value); + parent.del(key); + + if (++total % 10000 === 0) { + console.log('Migrated up %d undo blocks.', total); + await parent.write(); + parent = db.batch(); + } + }); + + console.log('Migrated all %d undo blocks.', total); + await parent.write(); +} + +async function migrateBlocks() { + console.log('Migrating blocks'); + + let parent = db.batch(); + + const iter = db.iterator({ + gte: layout.b.min(), + lte: layout.b.max(), + keys: true, + values: true + }); + + let total = 0; + + await iter.each(async (key, value) => { + const hash = key.slice(1); + await blockStore.write(hash, value); + parent.del(key); + + if (++total % 10000 === 0) { + console.log('Migrated up %d blocks.', total); + await parent.write(); + parent = db.batch(); + } + }); + + console.log('Migrated all %d blocks.', total); + await parent.write(); +} + +/* + * Execute + */ + +(async () => { + await db.open(); + await blockStore.ensure(); + await blockStore.open(); + + console.log('Opened %s.', process.argv[2]); + + await checkVersion(); + await migrateBlocks(); + await migrateUndoBlocks(); + await updateVersion(); + + console.log('Compacting database'); + await db.compactRange(); + await db.close(); + await blockStore.close(); +})().then(() => { + console.log('Migration complete.'); + process.exit(0); +}).catch((err) => { + console.error(err.stack); + process.exit(1); +}); diff --git a/test/blockstore-test.js b/test/blockstore-test.js new file mode 100644 index 00000000..39a777dc --- /dev/null +++ b/test/blockstore-test.js @@ -0,0 +1,1246 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const Logger = require('blgr'); +const bio = require('bufio'); +const assert = require('./util/assert'); +const common = require('./util/common'); +const {resolve} = require('path'); +const fs = require('bfile'); +const {rimraf, testdir} = require('./util/common'); +const random = require('bcrypto/lib/random'); + +const vectors = [ + common.readBlock('block300025'), + common.readBlock('block426884'), + common.readBlock('block898352') +]; + +const extra = [ + common.readBlock('block482683') +]; + +const undos = [ + common.readBlock('block300025'), + common.readBlock('block928816'), + common.readBlock('block928828'), + common.readBlock('block928831'), + common.readBlock('block928848'), + common.readBlock('block928849') +]; + +const { + AbstractBlockStore, + FileBlockStore, + LevelBlockStore +} = require('../lib/blockstore'); + +const layout = require('../lib/blockstore/layout'); +const {types} = require('../lib/blockstore/common'); + +const { + BlockRecord, + FileRecord +} = require('../lib/blockstore/records'); + +describe('BlockStore', function() { + describe('Abstract', function() { + let logger = null; + + function context(ctx) { + return {info: () => ctx}; + } + + beforeEach(() => { + logger = Logger.global; + Logger.global = {context}; + }); + + afterEach(() => { + Logger.global = logger; + }); + + it('construct with custom logger', async () => { + const store = new AbstractBlockStore({logger: {context}}); + assert(store.logger); + assert(store.logger.info); + assert.equal(store.logger.info(), 'blockstore'); + }); + + it('construct with default logger', async () => { + const store = new AbstractBlockStore(); + assert(store.logger); + assert(store.logger.info); + assert.equal(store.logger.info(), 'blockstore'); + }); + + it('has unimplemented base methods', async () => { + const methods = ['open', 'close', 'write', 'writeUndo', + 'read', 'readUndo', 'prune', 'pruneUndo', + 'has', 'hasUndo', 'ensure']; + + const store = new AbstractBlockStore(); + + for (const method of methods) { + assert(store[method]); + + let err = null; + try { + await store[method](); + } catch (e) { + err = e; + } + assert(err, `Expected unimplemented method ${method}.`); + assert.equal(err.message, 'Abstract method.'); + } + }); + }); + + describe('Records', function() { + describe('BlockRecord', function() { + function constructError(options) { + let err = null; + + try { + new BlockRecord({ + file: options.file, + position: options.position, + length: options.length + }); + } catch (e) { + err = e; + } + + assert(err); + } + + function toAndFromRaw(options) { + const rec1 = new BlockRecord(options); + assert.equal(rec1.file, options.file); + assert.equal(rec1.position, options.position); + assert.equal(rec1.length, options.length); + + const raw = rec1.toRaw(); + const rec2 = BlockRecord.fromRaw(raw); + assert.equal(rec2.file, options.file); + assert.equal(rec2.position, options.position); + assert.equal(rec2.length, options.length); + } + + it('construct with correct options', () => { + const rec = new BlockRecord({ + file: 12, + position: 23392, + length: 4194304 + }); + assert.equal(rec.file, 12); + assert.equal(rec.position, 23392); + assert.equal(rec.length, 4194304); + }); + + it('construct null record', () => { + const rec = new BlockRecord(); + assert.equal(rec.file, 0); + assert.equal(rec.position, 0); + assert.equal(rec.length, 0); + }); + + it('fail with signed number (file)', () => { + constructError({file: -1, position: 1, length: 1}); + }); + + it('fail with signed number (position)', () => { + constructError({file: 1, position: -1, length: 1}); + }); + + it('fail with signed number (length)', () => { + constructError({file: 1, position: 1, length: -1}); + }); + + it('fail with non-32-bit number (file)', () => { + constructError({file: Math.pow(2, 32), position: 1, length: 1}); + }); + + it('fail with non-32-bit number (position)', () => { + constructError({file: 1, position: Math.pow(2, 32), length: 1}); + }); + + it('fail with non-32-bit number (length)', () => { + constructError({file: 1, position: 1, length: Math.pow(2, 32)}); + }); + + it('construct with max 32-bit numbers', () => { + const max = Math.pow(2, 32) - 1; + + const rec = new BlockRecord({ + file: max, + position: max, + length: max + }); + + assert(rec); + assert.equal(rec.file, max); + assert.equal(rec.position, max); + assert.equal(rec.length, max); + }); + + it('serialize/deserialize file record (min)', () => { + toAndFromRaw({file: 0, position: 0, length: 0}); + }); + + it('serialize/deserialize file record', () => { + toAndFromRaw({file: 12, position: 23392, length: 4194304}); + }); + + it('serialize/deserialize file record (max)', () => { + const max = Math.pow(2, 32) - 1; + toAndFromRaw({file: max, position: max, length: max}); + }); + }); + + describe('FileRecord', function() { + function constructError(options) { + let err = null; + + try { + new FileRecord({ + blocks: options.blocks, + used: options.used, + length: options.length + }); + } catch (e) { + err = e; + } + + assert(err); + } + + function toAndFromRaw(options) { + const rec1 = new FileRecord(options); + assert.equal(rec1.blocks, options.blocks); + assert.equal(rec1.used, options.used); + assert.equal(rec1.length, options.length); + + const raw = rec1.toRaw(); + const rec2 = FileRecord.fromRaw(raw); + assert.equal(rec2.blocks, options.blocks); + assert.equal(rec2.used, options.used); + assert.equal(rec2.length, options.length); + } + + it('construct with correct options', () => { + const rec = new FileRecord({ + blocks: 1, + used: 4194304, + length: 20971520 + }); + assert.equal(rec.blocks, 1); + assert.equal(rec.used, 4194304); + assert.equal(rec.length, 20971520); + }); + + it('fail to with signed number (blocks)', () => { + constructError({blocks: -1, used: 1, length: 1}); + }); + + it('fail to with signed number (used)', () => { + constructError({blocks: 1, used: -1, length: 1}); + }); + + it('fail to with signed number (length)', () => { + constructError({blocks: 1, used: 1, length: -1}); + }); + + it('fail to with non-32-bit number (blocks)', () => { + constructError({blocks: Math.pow(2, 32), used: 1, length: 1}); + }); + + it('fail to with non-32-bit number (used)', () => { + constructError({blocks: 1, used: Math.pow(2, 32), length: 1}); + }); + + it('fail to with non-32-bit number (length)', () => { + constructError({blocks: 1, used: 1, length: Math.pow(2, 32)}); + }); + + it('serialize/deserialize block record (min)', () => { + toAndFromRaw({blocks: 0, used: 0, length: 0}); + }); + + it('serialize/deserialize block record', () => { + toAndFromRaw({blocks: 10, used: 4194304, length: 20971520}); + }); + + it('serialize/deserialize block record (max)', () => { + const max = Math.pow(2, 32) - 1; + toAndFromRaw({blocks: max, used: max, length: max}); + }); + }); + }); + + describe('FileBlockStore (Unit)', function() { + const location = '/tmp/.bcoin/blocks'; + let store = null; + + before(() => { + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + }); + + describe('constructor', function() { + it('will pass options to super', () => { + const info = () => 'info'; + const logger = { + context: () => { + return {info}; + } + }; + + const store = new FileBlockStore({ + location: '/tmp/.bcoin/blocks', + maxFileLength: 1024, + logger: logger + }); + + assert.strictEqual(store.logger.info, info); + }); + + it('will error with invalid location', () => { + let err = null; + + try { + new FileBlockStore({ + location: 'tmp/.bcoin/blocks', + maxFileLength: 1024 + }); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Location not absolute.'); + }); + + it('will error with invalid max file length', () => { + let err = null; + + try { + new FileBlockStore({ + location: location, + maxFileLength: 'notanumber' + }); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Invalid max file length.'); + }); + }); + + describe('allocate', function() { + it('will fail with length above file max', async () => { + let err = null; + try { + await store.allocate(types.BLOCK, 1025); + } catch (e) { + err = e; + } + assert(err); + assert.equal(err.message, 'Block length above max file length.'); + }); + }); + + describe('filepath', function() { + it('will give correct path (0)', () => { + const filepath = store.filepath(types.BLOCK, 0); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00000.dat'); + }); + + it('will give correct path (1)', () => { + const filepath = store.filepath(types.BLOCK, 7); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00007.dat'); + }); + + it('will give correct path (2)', () => { + const filepath = store.filepath(types.BLOCK, 23); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00023.dat'); + }); + + it('will give correct path (3)', () => { + const filepath = store.filepath(types.BLOCK, 456); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk00456.dat'); + }); + + it('will give correct path (4)', () => { + const filepath = store.filepath(types.BLOCK, 8999); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk08999.dat'); + }); + + it('will give correct path (5)', () => { + const filepath = store.filepath(types.BLOCK, 99999); + assert.equal(filepath, '/tmp/.bcoin/blocks/blk99999.dat'); + }); + + it('will fail over max size', () => { + let err = null; + try { + store.filepath(types.BLOCK, 100000); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'File number too large.'); + }); + + it('will give undo type', () => { + const filepath = store.filepath(types.UNDO, 99999); + assert.equal(filepath, '/tmp/.bcoin/blocks/blu99999.dat'); + }); + + it('will fail for unknown prefix', () => { + let err = null; + try { + store.filepath(0, 1234); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Unknown file prefix.'); + }); + }); + + describe('write', function() { + const write = fs.write; + const open = fs.open; + const close = fs.close; + let allocate = null; + let has = null; + + beforeEach(() => { + allocate = store.allocate; + has = store.db.has; + }); + + afterEach(() => { + // Restore stubbed methods. + fs.write = write; + fs.open = open; + fs.close = close; + store.allocate = allocate; + store.db.has = has; + }); + + it('will error if total magic bytes not written', async () => { + let err = null; + + store.allocate = () => { + return { + fileno: 20, + filerecord: { + used: 0 + }, + filepath: '/tmp/.bcoin/blocks/blk00020.dat' + }; + }; + store.db.has = () => false; + fs.open = () => 7; + fs.close = () => undefined; + fs.write = () => 0; + + try { + const hash = random.randomBytes(128); + const block = random.randomBytes(32); + await store.write(hash, block); + } catch (e) { + err = e; + } + + assert(err, 'Expected error.'); + assert.equal(err.message, 'Could not write block magic.'); + }); + + it('will error if total block bytes not written', async () => { + let err = 0; + + let called = 0; + store.allocate = () => { + return { + fileno: 20, + filerecord: { + used: 0 + }, + filepath: '/tmp/.bcoin/blocks/blk00020.dat' + }; + }; + store.db.has = () => false; + fs.open = () => 7; + fs.close = () => undefined; + fs.write = (fd, buffer, offset, length, position) => { + let written = 0; + + if (called === 0) + written = length; + + called += 1; + + return written; + }; + + try { + const hash = random.randomBytes(128); + const block = random.randomBytes(32); + await store.write(hash, block); + } catch (e) { + err = e; + } + + assert(err, 'Expected error.'); + assert.equal(err.message, 'Could not write block.'); + }); + + it('will close file if write throws', async () => { + let err = null; + let closed = null; + + store.allocate = () => { + return { + fileno: 20, + filerecord: { + used: 0 + }, + filepath: '/tmp/.bcoin/blocks/blk00020.dat' + }; + }; + store.db.has = () => false; + fs.open = () => 7; + fs.close = (fd) => { + closed = fd; + }; + fs.write = () => { + throw new Error('Test.'); + }; + + try { + const hash = random.randomBytes(128); + const block = random.randomBytes(32); + await store.write(hash, block); + } catch (e) { + err = e; + } + + assert(err, 'Expected error.'); + assert.equal(err.message, 'Test.'); + assert.equal(closed, 7); + }); + }); + + describe('read', function() { + const read = fs.read; + const open = fs.open; + const close = fs.close; + let get = null; + let raw = null; + + before(() => { + const record = new BlockRecord({ + file: 1, + position: 8, + length: 100 + }); + raw = record.toRaw(); + }); + + beforeEach(() => { + get = store.db.get; + }); + + afterEach(() => { + // Restore stubbed methods. + store.db.get = get; + fs.read = read; + fs.open = open; + fs.close = close; + }); + + it('will error if total read bytes not correct', async () => { + let err = null; + + store.db.get = () => raw; + fs.open = () => 7; + fs.close = () => undefined; + fs.read = () => 99; + + try { + const hash = random.randomBytes(128); + const block = random.randomBytes(32); + await store.read(hash, block); + } catch (e) { + err = e; + } + + assert(err, 'Expected error.'); + assert.equal(err.message, 'Wrong number of bytes read.'); + }); + + it('will close file if read throws', async () => { + let err = null; + let closed = null; + + store.db.get = () => raw; + fs.open = () => 7; + fs.close = (fd) => { + closed = fd; + }; + fs.read = () => { + throw new Error('Test.'); + }; + + try { + const hash = random.randomBytes(128); + const block = random.randomBytes(32); + await store.read(hash, block); + } catch (e) { + err = e; + } + + assert(err, 'Expected error.'); + assert.equal(err.message, 'Test.'); + assert.equal(closed, 7); + }); + }); + }); + + describe('FileBlockStore (Integration 1)', function() { + const location = testdir('blockstore'); + let store = null; + + beforeEach(async () => { + await rimraf(location); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + + await store.ensure(); + await store.open(); + }); + + afterEach(async () => { + await store.close(); + }); + + after(async () => { + await rimraf(location); + }); + + it('will write and read a block', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const block2 = await store.read(hash); + + assert.bufferEqual(block1, block2); + }); + + it('will write and read block undo coins', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.writeUndo(hash, block1); + + const block2 = await store.readUndo(hash); + + assert.bufferEqual(block1, block2); + }); + + it('will read a block w/ offset and length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 15; + + const block2 = await store.read(hash, offset, size); + + assert.bufferEqual(block1.slice(offset, offset + size), block2); + }); + + it('will fail to read w/ out-of-bounds length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 50; + + let err = null; + try { + await store.read(hash, offset, size); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Out-of-bounds read.'); + }); + + it('will allocate new files', async () => { + const blocks = []; + + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + blocks.push({hash, block}); + await store.write(hash, block); + const block2 = await store.read(hash); + assert.bufferEqual(block2, block); + } + + const first = await fs.stat(store.filepath(types.BLOCK, 0)); + const second = await fs.stat(store.filepath(types.BLOCK, 1)); + const third = await fs.stat(store.filepath(types.BLOCK, 2)); + assert.equal(first.size, 952); + assert.equal(second.size, 952); + assert.equal(third.size, 272); + + const magic = (8 * 16); + const len = first.size + second.size + third.size - magic; + assert.equal(len, 128 * 16); + + for (let i = 0; i < 16; i++) { + const expect = blocks[i]; + const block = await store.read(expect.hash); + assert.bufferEqual(block, expect.block); + } + }); + + it('will allocate new files with block undo coins', async () => { + const blocks = []; + + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + blocks.push({hash, block}); + await store.writeUndo(hash, block); + const block2 = await store.readUndo(hash); + assert.bufferEqual(block2, block); + } + + const first = await fs.stat(store.filepath(types.UNDO, 0)); + const second = await fs.stat(store.filepath(types.UNDO, 1)); + const third = await fs.stat(store.filepath(types.UNDO, 2)); + + const magic = (40 * 16); + const len = first.size + second.size + third.size - magic; + assert.equal(len, 128 * 16); + + for (let i = 0; i < 16; i++) { + const expect = blocks[i]; + const block = await store.readUndo(expect.hash); + assert.bufferEqual(block, expect.block); + } + }); + + it('will recover from interrupt during block write', async () => { + { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + + const block2 = await store.read(hash); + assert.bufferEqual(block2, block); + } + + // Manually insert a partially written block to the + // end of file as would be the case of an untimely + // interrupted write of a block. The file record + // would not be updated to include the used bytes and + // thus this data should be overwritten. + { + const filepath = store.filepath(types.BLOCK, 0); + + const fd = await fs.open(filepath, 'a'); + + const bw = bio.write(8); + bw.writeU32(store.network.magic); + bw.writeU32(73); + const magic = bw.render(); + + const failblock = random.randomBytes(73); + + const mwritten = await fs.write(fd, magic, 0, 8); + const bwritten = await fs.write(fd, failblock, 0, 73); + + await fs.close(fd); + + assert.equal(mwritten, 8); + assert.equal(bwritten, 73); + } + + // Now check that this block has the correct position + // in the file and that it can be read correctly. + { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + + const block2 = await store.read(hash); + assert.bufferEqual(block2, block); + } + }); + + it('will not write blocks at the same position', (done) => { + let err = null; + let finished = 0; + + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + + // Accidentally don't use `await` and attempt to + // write multiple blocks in parallel and at the + // same file position. + const promise = store.write(hash, block); + promise.catch((e) => { + err = e; + }).finally(() => { + finished += 1; + if (finished >= 16) { + assert(err); + assert(err.message, 'Already writing.'); + done(); + } + }); + } + }); + + it('will not duplicate a block on disk', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + + const first = await store.write(hash, block); + assert.equal(first, true); + const second = await store.write(hash, block); + assert.equal(second, false); + + const pruned = await store.prune(hash); + assert.equal(pruned, true); + + assert.equal(await fs.exists(store.filepath(types.BLOCK, 0)), false); + }); + + it('will return null if block not found', async () => { + const hash = random.randomBytes(32); + const block = await store.read(hash); + assert.strictEqual(block, null); + }); + + it('will check if block exists (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + }); + + it('will check if block exists (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + const exists = await store.has(hash); + assert.strictEqual(exists, true); + }); + + it('will check if block undo coins exists (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.hasUndo(hash); + assert.strictEqual(exists, false); + }); + + it('will check if block undo coins exists (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.writeUndo(hash, block); + const exists = await store.hasUndo(hash); + assert.strictEqual(exists, true); + }); + + it('will prune blocks', async () => { + const hashes = []; + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + hashes.push(hash); + await store.write(hash, block); + } + + const first = await fs.stat(store.filepath(types.BLOCK, 0)); + const second = await fs.stat(store.filepath(types.BLOCK, 1)); + const third = await fs.stat(store.filepath(types.BLOCK, 2)); + + const magic = (8 * 16); + const len = first.size + second.size + third.size - magic; + assert.equal(len, 128 * 16); + + for (let i = 0; i < 16; i++) { + const pruned = await store.prune(hashes[i]); + assert.strictEqual(pruned, true); + } + + assert.equal(await fs.exists(store.filepath(types.BLOCK, 0)), false); + assert.equal(await fs.exists(store.filepath(types.BLOCK, 1)), false); + assert.equal(await fs.exists(store.filepath(types.BLOCK, 2)), false); + + for (let i = 0; i < 16; i++) { + const exists = await store.has(hashes[i]); + assert.strictEqual(exists, false); + } + + const exists = await store.db.has(layout.f.encode(types.BLOCK, 0)); + assert.strictEqual(exists, false); + }); + + it('will prune block undo coins', async () => { + const hashes = []; + for (let i = 0; i < 16; i++) { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + hashes.push(hash); + await store.writeUndo(hash, block); + } + + const first = await fs.stat(store.filepath(types.UNDO, 0)); + const second = await fs.stat(store.filepath(types.UNDO, 1)); + const third = await fs.stat(store.filepath(types.UNDO, 2)); + + const magic = (40 * 16); + const len = first.size + second.size + third.size - magic; + assert.equal(len, 128 * 16); + + for (let i = 0; i < 16; i++) { + const pruned = await store.pruneUndo(hashes[i]); + assert.strictEqual(pruned, true); + } + + assert.equal(await fs.exists(store.filepath(types.UNDO, 0)), false); + assert.equal(await fs.exists(store.filepath(types.UNDO, 1)), false); + assert.equal(await fs.exists(store.filepath(types.UNDO, 2)), false); + + for (let i = 0; i < 16; i++) { + const exists = await store.hasUndo(hashes[i]); + assert.strictEqual(exists, false); + } + + const exists = await store.db.has(layout.f.encode(types.UNDO, 0)); + assert.strictEqual(exists, false); + }); + }); + + describe('FileBlockStore (Integration 2)', function() { + const location = testdir('blockstore'); + let store = null; + + beforeEach(async () => { + await rimraf(location); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 * 1024 + }); + + await store.ensure(); + await store.open(); + }); + + afterEach(async () => { + await store.close(); + }); + + after(async () => { + await rimraf(location); + }); + + it('will import from files (e.g. db corruption)', async () => { + const blocks = []; + + for (let i = 0; i < vectors.length; i++) { + const [block] = vectors[i].getBlock(); + const hash = block.hash(); + const raw = block.toRaw(); + + blocks.push({hash, block: raw}); + await store.write(hash, raw); + } + + await store.close(); + + await rimraf(resolve(location, './index')); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + + await store.open(); + + for (let i = 0; i < vectors.length; i++) { + const expect = blocks[i]; + const block = await store.read(expect.hash); + assert.equal(block.length, expect.block.length); + assert.bufferEqual(block, expect.block); + } + }); + + it('will import from files after write interrupt', async () => { + const blocks = []; + + for (let i = 0; i < vectors.length; i++) { + const [block] = vectors[i].getBlock(); + const hash = block.hash(); + const raw = block.toRaw(); + + blocks.push({hash, block: raw}); + await store.write(hash, raw); + } + + await store.close(); + + assert.equal(await fs.exists(store.filepath(types.BLOCK, 0)), true); + assert.equal(await fs.exists(store.filepath(types.BLOCK, 1)), true); + assert.equal(await fs.exists(store.filepath(types.BLOCK, 2)), false); + + // Write partial block as would be the case in a + // block write interrupt. + const [partial] = extra[0].getBlock(); + { + // Include all of the header, but not the block. + let raw = partial.toRaw(); + const actual = raw.length; + const part = raw.length - 1; + raw = raw.slice(0, part); + + const filepath = store.filepath(types.BLOCK, 1); + + const fd = await fs.open(filepath, 'a'); + + const bw = bio.write(8); + bw.writeU32(store.network.magic); + bw.writeU32(actual); + const magic = bw.render(); + + const mwritten = await fs.write(fd, magic, 0, 8); + const bwritten = await fs.write(fd, raw, 0, part); + + await fs.close(fd); + + assert.equal(mwritten, 8); + assert.equal(bwritten, part); + } + + await rimraf(resolve(location, './index')); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + + await store.open(); + + const incomplete = await store.read(partial.hash()); + assert(incomplete === null); + + for (let i = 0; i < vectors.length; i++) { + const expect = blocks[i]; + const block = await store.read(expect.hash); + assert.equal(block.length, expect.block.length); + assert.bufferEqual(block, expect.block); + } + }); + + it('will import undo blocks from files', async () => { + const blocks = []; + + for (let i = 0; i < undos.length; i++) { + const [block] = undos[i].getBlock(); + const raw = undos[i].undoRaw; + const hash = block.hash(); + + blocks.push({hash, block: raw}); + await store.writeUndo(hash, raw); + } + + await store.close(); + + await rimraf(resolve(location, './index')); + + store = new FileBlockStore({ + location: location, + maxFileLength: 1024 + }); + + await store.open(); + + for (let i = 0; i < undos.length; i++) { + const expect = blocks[i]; + const block = await store.readUndo(expect.hash); + assert.equal(block.length, expect.block.length); + assert.bufferEqual(block, expect.block); + } + }); + }); + + describe('LevelBlockStore', function() { + const location = testdir('blockstore'); + let store = null; + + beforeEach(async () => { + await rimraf(location); + + store = new LevelBlockStore({ + location: location + }); + + await store.ensure(); + await store.open(); + }); + + afterEach(async () => { + await store.close(); + }); + + after(async () => { + await rimraf(location); + }); + + it('will write and read a block', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const block2 = await store.read(hash); + + assert.bufferEqual(block1, block2); + }); + + it('will write and read block undo coins', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.writeUndo(hash, block1); + + const block2 = await store.readUndo(hash); + + assert.bufferEqual(block1, block2); + }); + + it('will read a block w/ offset and length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 15; + + const block2 = await store.read(hash, offset, size); + + assert.bufferEqual(block1.slice(offset, offset + size), block2); + }); + + it('will fail to read w/ out-of-bounds length', async () => { + const block1 = random.randomBytes(128); + const hash = random.randomBytes(32); + + await store.write(hash, block1); + + const offset = 79; + const size = 50; + + let err = null; + try { + await store.read(hash, offset, size); + } catch (e) { + err = e; + } + + assert(err); + assert.equal(err.message, 'Out-of-bounds read.'); + }); + + it('will check if block exists (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + }); + + it('will check if block exists (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + const exists = await store.has(hash); + assert.strictEqual(exists, true); + }); + + it('will check if block undo coins exists (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + }); + + it('will check if block undo coins exists (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.writeUndo(hash, block); + const exists = await store.hasUndo(hash); + assert.strictEqual(exists, true); + }); + + it('will prune blocks (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.write(hash, block); + const pruned = await store.prune(hash); + assert.strictEqual(pruned, true); + const block2 = await store.read(hash); + assert.strictEqual(block2, null); + }); + + it('will prune blocks (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.has(hash); + assert.strictEqual(exists, false); + const pruned = await store.prune(hash); + assert.strictEqual(pruned, false); + }); + + it('will prune block undo coins (true)', async () => { + const block = random.randomBytes(128); + const hash = random.randomBytes(32); + await store.writeUndo(hash, block); + const pruned = await store.pruneUndo(hash); + assert.strictEqual(pruned, true); + const block2 = await store.readUndo(hash); + assert.strictEqual(block2, null); + }); + + it('will prune block undo coins (false)', async () => { + const hash = random.randomBytes(32); + const exists = await store.hasUndo(hash); + assert.strictEqual(exists, false); + const pruned = await store.pruneUndo(hash); + assert.strictEqual(pruned, false); + }); + }); +}); diff --git a/test/chain-test.js b/test/chain-test.js index d50237ac..9c915031 100644 --- a/test/chain-test.js +++ b/test/chain-test.js @@ -17,6 +17,7 @@ const Output = require('../lib/primitives/output'); const common = require('../lib/blockchain/common'); const nodejsUtil = require('util'); const Opcode = require('../lib/script/opcode'); +const BlockStore = require('../lib/blockstore/level'); const opcodes = Script.opcodes; const ZERO_KEY = Buffer.alloc(33, 0x00); @@ -30,8 +31,14 @@ const workers = new WorkerPool({ enabled: true }); +const blocks = new BlockStore({ + memory: true, + network +}); + const chain = new Chain({ memory: true, + blocks, network, workers }); @@ -115,6 +122,7 @@ describe('Chain', function() { this.timeout(process.browser ? 1200000 : 60000); it('should open chain and miner', async () => { + await blocks.open(); await chain.open(); await miner.open(); }); @@ -895,5 +903,6 @@ describe('Chain', function() { it('should cleanup', async () => { await miner.close(); await chain.close(); + await blocks.close(); }); }); diff --git a/test/mempool-test.js b/test/mempool-test.js index ed76f869..247720f6 100644 --- a/test/mempool-test.js +++ b/test/mempool-test.js @@ -18,6 +18,7 @@ const Script = require('../lib/script/script'); const opcodes = Script.opcodes; const Witness = require('../lib/script/witness'); const MemWallet = require('./util/memwallet'); +const BlockStore = require('../lib/blockstore/level'); const ALL = Script.hashType.ALL; const ONE_HASH = Buffer.alloc(32, 0x00); @@ -27,9 +28,14 @@ const workers = new WorkerPool({ enabled: true }); +const blocks = new BlockStore({ + memory: true +}); + const chain = new Chain({ memory: true, - workers + workers, + blocks }); const mempool = new Mempool({ @@ -68,6 +74,7 @@ describe('Mempool', function() { it('should open mempool', async () => { await workers.open(); + await blocks.open(); await chain.open(); await mempool.open(); chain.state.flags |= Script.flags.VERIFY_WITNESS; @@ -453,6 +460,7 @@ describe('Mempool', function() { it('should destroy mempool', async () => { await mempool.close(); await chain.close(); + await blocks.close(); await workers.close(); }); }); diff --git a/test/pow-test.js b/test/pow-test.js index bd2707f5..6f54ad5a 100644 --- a/test/pow-test.js +++ b/test/pow-test.js @@ -7,6 +7,7 @@ const assert = require('./util/assert'); const Chain = require('../lib/blockchain/chain'); const ChainEntry = require('../lib/blockchain/chainentry'); const Network = require('../lib/protocol/network'); +const BlockStore = require('../lib/blockstore/level'); const network = Network.get('main'); @@ -14,13 +15,20 @@ function random(max) { return Math.floor(Math.random() * max); } -const chain = new Chain({ +const blocks = new BlockStore({ memory: true, network }); +const chain = new Chain({ + memory: true, + network, + blocks +}); + describe('Difficulty', function() { it('should open chain', async () => { + await blocks.open(); await chain.open(); }); diff --git a/test/util/common.js b/test/util/common.js index 9928eb45..39177fc5 100644 --- a/test/util/common.js +++ b/test/util/common.js @@ -1,9 +1,11 @@ 'use strict'; const assert = require('assert'); +const {tmpdir} = require('os'); const path = require('path'); const fs = require('bfile'); const bio = require('bufio'); +const {randomBytes} = require('bcrypto/lib/random'); const Block = require('../../lib/primitives/block'); const MerkleBlock = require('../../lib/primitives/merkleblock'); const Headers = require('../../lib/primitives/headers'); @@ -85,6 +87,21 @@ common.writeTX = function writeTX(name, tx, view) { common.writeFile(`${name}-undo.raw`, undoRaw); }; +common.testdir = function(name) { + assert(/^[a-z]+$/.test(name), 'Invalid name'); + + const uniq = randomBytes(4).toString('hex'); + return path.join(tmpdir(), `bcoin-test-${name}-${uniq}`); +}; + +common.rimraf = async function(p) { + const allowed = /bcoin\-test\-[a-z]+\-[a-f0-9]{8}(\/[a-z]+)?$/; + if (!allowed.test(p)) + throw new Error(`Path not allowed: ${p}.`); + + return await fs.rimraf(p); +}; + function parseUndo(data) { const br = bio.read(data); const items = [];