RanchiMall/fcoin
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

db: change iterator api to be more loop-friendly.

Browse Source
This commit is contained in:
Christopher Jeffrey 2017-10-15 02:17:34 -07:00
parent 1c1e429383
commit 4d8ca8c16f
No known key found for this signature in database
GPG Key ID: 8962AB9DE6666BBD
8 changed files with 183 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/blockchain/chaindb.js
View File

@ -1881,7 +1881,9 @@ ChainDB.prototype.pruneBlock = async function pruneBlock(entry) {
ChainDB.prototype.saveFlags = function saveFlags() { ChainDB.prototype.saveFlags = function saveFlags() {
const flags = ChainFlags.fromOptions(this.options); const flags = ChainFlags.fromOptions(this.options);
return this.db.put(layout.O, flags.toRaw()); const batch = this.db.batch();
batch.put(layout.O, flags.toRaw());
return batch.write();
}; };
/** /**

80
lib/db/level.js
View File

@ -17,37 +17,19 @@ DB.prototype.close = function close(callback) {
}; };
DB.prototype.get = function get(key, options, callback) { DB.prototype.get = function get(key, options, callback) {
if (this.bufferKeys && Buffer.isBuffer(key)) this.level.get(toHex(key), options, callback);
key = key.toString('hex');
this.level.get(key, options, callback);
}; };
DB.prototype.put = function put(key, value, options, callback) { DB.prototype.put = function put(key, value, options, callback) {
if (this.bufferKeys && Buffer.isBuffer(key)) this.level.put(toHex(key), value, options, callback);
key = key.toString('hex');
this.level.put(key, value, options, callback);
}; };
DB.prototype.del = function del(key, options, callback) { DB.prototype.del = function del(key, options, callback) {
if (this.bufferKeys && Buffer.isBuffer(key)) this.level.del(toHex(key), options, callback);
key = key.toString('hex');
this.level.del(key, options, callback);
}; };
DB.prototype.batch = function batch(ops, options, callback) { DB.prototype.batch = function batch() {
if (!ops) return new Batch(this);
return new Batch(this);
if (this.bufferKeys) {
for (const op of ops) {
if (Buffer.isBuffer(op.key))
op.key = op.key.toString('hex');
}
}
this.level.batch(ops, options, callback);
return undefined;
}; };
DB.prototype.iterator = function iterator(options) { DB.prototype.iterator = function iterator(options) {
@ -65,17 +47,13 @@ function Batch(db) {
} }
Batch.prototype.put = function put(key, value) { Batch.prototype.put = function put(key, value) {
if (this.db.bufferKeys && Buffer.isBuffer(key)) this.batch.put(toHex(key), value);
key = key.toString('hex');
this.batch.put(key, value);
this.hasOps = true; this.hasOps = true;
return this; return this;
}; };
Batch.prototype.del = function del(key) { Batch.prototype.del = function del(key) {
if (this.db.bufferKeys && Buffer.isBuffer(key)) this.batch.del(toHex(key));
key = key.toString('hex');
this.batch.del(key);
this.hasOps = true; this.hasOps = true;
return this; return this;
}; };
@ -93,20 +71,22 @@ Batch.prototype.clear = function clear() {
}; };
function Iterator(db, options) { function Iterator(db, options) {
if (db.bufferKeys) { const opt = {
if (Buffer.isBuffer(options.gt)) gt: toHex(options.gt),
options.gt = options.gt.toString('hex'); gte: toHex(options.gte),
if (Buffer.isBuffer(options.gte)) lt: toHex(options.lt),
options.gte = options.gte.toString('hex'); lte: toHex(options.lte),
if (Buffer.isBuffer(options.lt)) limit: options.limit,
options.lt = options.lt.toString('hex'); reverse: options.reverse,
if (Buffer.isBuffer(options.lte)) keys: options.keys,
options.lte = options.lte.toString('hex'); values: options.values,
} keyAsBuffer: false,
options.keyAsBuffer = false; valueAsBuffer: true
};
this.db = db; this.db = db;
this.iter = db.level.iterator(options); this.iter = db.level.iterator(opt);
this._end = false; this.ended = false;
} }
Iterator.prototype.next = function next(callback) { Iterator.prototype.next = function next(callback) {
@ -114,7 +94,7 @@ Iterator.prototype.next = function next(callback) {
// Hack for level-js: it doesn't actually // Hack for level-js: it doesn't actually
// end iterators -- it keeps streaming keys // end iterators -- it keeps streaming keys
// and values. // and values.
if (this._end) if (this.ended)
return; return;
if (err) { if (err) {
@ -138,18 +118,22 @@ Iterator.prototype.next = function next(callback) {
}; };
Iterator.prototype.seek = function seek(key) { Iterator.prototype.seek = function seek(key) {
if (this.db.bufferKeys && Buffer.isBuffer(key)) this.iter.seek(toHex(key));
key = key.toString('hex');
this.iter.seek(key);
}; };
Iterator.prototype.end = function end(callback) { Iterator.prototype.end = function end(callback) {
if (this._end) { if (this.ended) {
callback(new Error('end() already called on iterator.')); callback(new Error('end() already called on iterator.'));
return; return;
} }
this._end = true; this.ended = true;
this.iter.end(callback); this.iter.end(callback);
}; };
function toHex(key) {
if (Buffer.isBuffer(key))
return key.toString('hex');
return key;
}
module.exports = DB; module.exports = DB;

221
lib/db/lowlevelup.js
View File

@ -8,8 +8,6 @@
'use strict'; 'use strict';
const assert = require('assert'); const assert = require('assert');
const Lock = require('../utils/lock');
const co = require('../utils/co');
const LOW = Buffer.from([0x00]); const LOW = Buffer.from([0x00]);
const HIGH = Buffer.from([0xff]); const HIGH = Buffer.from([0xff]);
@ -39,7 +37,6 @@ function LowlevelUp(backend, location, options) {
this.options = new LLUOptions(options); this.options = new LLUOptions(options);
this.backend = backend; this.backend = backend;
this.location = location; this.location = location;
this.locker = new Lock();
this.loading = false; this.loading = false;
this.closing = false; this.closing = false;
@ -89,27 +86,10 @@ LowlevelUp.prototype.init = function init() {
/** /**
* Open the database. * Open the database.
* @method
* @returns {Promise} * @returns {Promise}
*/ */
LowlevelUp.prototype.open = async function open() { LowlevelUp.prototype.open = async function open() {
const unlock = await this.locker.lock();
try {
return await this._open();
} finally {
unlock();
}
};
/**
* Open the database (without a lock).
* @method
* @private
* @returns {Promise}
*/
LowlevelUp.prototype._open = async function _open() {
if (this.loaded) if (this.loaded)
throw new Error('Database is already open.'); throw new Error('Database is already open.');
@ -131,27 +111,10 @@ LowlevelUp.prototype._open = async function _open() {
/** /**
* Close the database. * Close the database.
* @method
* @returns {Promise} * @returns {Promise}
*/ */
LowlevelUp.prototype.close = async function close() { LowlevelUp.prototype.close = async function close() {
const unlock = await this.locker.lock();
try {
return await this._close();
} finally {
unlock();
}
};
/**
* Close the database (without a lock).
* @method
* @private
* @returns {Promise}
*/
LowlevelUp.prototype._close = async function _close() {
if (!this.loaded) if (!this.loaded)
throw new Error('Database is already closed.'); throw new Error('Database is already closed.');
@ -180,7 +143,7 @@ LowlevelUp.prototype._close = async function _close() {
LowlevelUp.prototype.load = function load() { LowlevelUp.prototype.load = function load() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.binding.open(this.options, co.wrap(resolve, reject)); this.binding.open(this.options, wrap(resolve, reject));
}); });
}; };
@ -192,7 +155,7 @@ LowlevelUp.prototype.load = function load() {
LowlevelUp.prototype.unload = function unload() { LowlevelUp.prototype.unload = function unload() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.binding.close(co.wrap(resolve, reject)); this.binding.close(wrap(resolve, reject));
}); });
}; };
@ -213,7 +176,7 @@ LowlevelUp.prototype.destroy = function destroy() {
return; return;
} }
this.backend.destroy(this.location, co.wrap(resolve, reject)); this.backend.destroy(this.location, wrap(resolve, reject));
}); });
}; };
@ -234,7 +197,7 @@ LowlevelUp.prototype.repair = function repair() {
return; return;
} }
this.backend.repair(this.location, co.wrap(resolve, reject)); this.backend.repair(this.location, wrap(resolve, reject));
}); });
}; };
@ -253,7 +216,7 @@ LowlevelUp.prototype.backup = function backup(path) {
reject(new Error('Database is closed.')); reject(new Error('Database is closed.'));
return; return;
} }
this.binding.backup(path, co.wrap(resolve, reject)); this.binding.backup(path, wrap(resolve, reject));
}); });
}; };
@ -299,7 +262,7 @@ LowlevelUp.prototype.put = function put(key, value) {
reject(new Error('Database is closed.')); reject(new Error('Database is closed.'));
return; return;
} }
this.binding.put(key, value, co.wrap(resolve, reject)); this.binding.put(key, value, wrap(resolve, reject));
}); });
}; };
@ -315,30 +278,20 @@ LowlevelUp.prototype.del = function del(key) {
reject(new Error('Database is closed.')); reject(new Error('Database is closed.'));
return; return;
} }
this.binding.del(key, co.wrap(resolve, reject)); this.binding.del(key, wrap(resolve, reject));
}); });
}; };
/** /**
* Create an atomic batch. * Create an atomic batch.
* @param {Array?} ops
* @returns {Batch} * @returns {Batch}
*/ */
LowlevelUp.prototype.batch = function batch(ops) { LowlevelUp.prototype.batch = function batch() {
if (!ops) { if (!this.loaded)
if (!this.loaded) throw new Error('Database is closed.');
throw new Error('Database is closed.');
return new Batch(this);
}
return new Promise((resolve, reject) => { return new Batch(this);
if (!this.loaded) {
reject(new Error('Database is closed.'));
return;
}
this.binding.batch(ops, co.wrap(resolve, reject));
});
}; };
/** /**
@ -389,7 +342,7 @@ LowlevelUp.prototype.approximateSize = function approximateSize(start, end) {
return; return;
} }
this.binding.approximateSize(start, end, co.wrap(resolve, reject)); this.binding.approximateSize(start, end, wrap(resolve, reject));
}); });
}; };
@ -418,7 +371,7 @@ LowlevelUp.prototype.compactRange = function compactRange(start, end) {
return; return;
} }
this.binding.compactRange(start, end, co.wrap(resolve, reject)); this.binding.compactRange(start, end, wrap(resolve, reject));
}); });
}; };
@ -442,7 +395,6 @@ LowlevelUp.prototype.has = async function has(key) {
*/ */
LowlevelUp.prototype.range = async function range(options) { LowlevelUp.prototype.range = async function range(options) {
const items = [];
const parse = options.parse; const parse = options.parse;
const iter = this.iterator({ const iter = this.iterator({
@ -452,23 +404,28 @@ LowlevelUp.prototype.range = async function range(options) {
values: true values: true
}); });
for (;;) { const items = [];
let item = await iter.next();
if (!item) while (await iter.next()) {
break; const {key, value} = iter;
if (parse) { if (parse) {
let item;
try { try {
item = parse(item.key, item.value); item = parse(key, value);
} catch (e) { } catch (e) {
await iter.end(); await iter.end();
throw e; throw e;
} }
if (item)
items.push(item);
continue;
} }
if (item) items.push(new IteratorItem(key, value));
items.push(item);
} }
return items; return items;
@ -482,7 +439,6 @@ LowlevelUp.prototype.range = async function range(options) {
*/ */
LowlevelUp.prototype.keys = async function keys(options) { LowlevelUp.prototype.keys = async function keys(options) {
const items = [];
const parse = options.parse; const parse = options.parse;
const iter = this.iterator({ const iter = this.iterator({
@ -492,13 +448,10 @@ LowlevelUp.prototype.keys = async function keys(options) {
values: false values: false
}); });
for (;;) { const items = [];
const item = await iter.next();
if (!item) while (await iter.next()) {
break; let {key} = iter;
let key = item.key;
if (parse) { if (parse) {
try { try {
@ -534,13 +487,8 @@ LowlevelUp.prototype.values = async function values(options) {
values: true values: true
}); });
for (;;) { while (await iter.next()) {
const item = await iter.next(); let {value} = iter;
if (!item)
break;
let value = item.value;
if (parse) { if (parse) {
try { try {
@ -589,18 +537,20 @@ LowlevelUp.prototype.dump = async function dump() {
*/ */
LowlevelUp.prototype.checkVersion = async function checkVersion(key, version) { LowlevelUp.prototype.checkVersion = async function checkVersion(key, version) {
let data = await this.get(key); const data = await this.get(key);
if (!data) { if (!data) {
data = Buffer.allocUnsafe(4); const value = Buffer.allocUnsafe(4);
data.writeUInt32LE(version, 0, true); value.writeUInt32LE(version, 0, true);
await this.put(key, data); const batch = this.batch();
batch.put(key, value);
await batch.write();
return; return;
} }
data = data.readUInt32LE(0, true); const num = data.readUInt32LE(0, true);
if (data !== version) if (num !== version)
throw new Error(VERSION_ERROR); throw new Error(VERSION_ERROR);
}; };
@ -625,22 +575,21 @@ LowlevelUp.prototype.clone = async function clone(path) {
await tmp.open(); await tmp.open();
let batch = tmp.batch();
let total = 0;
const iter = this.iterator({ const iter = this.iterator({
keys: true, keys: true,
values: true values: true
}); });
for (;;) { let batch = tmp.batch();
const item = await iter.next(); let total = 0;
if (!item) while (await iter.next()) {
break; const {key, value} = iter;
batch.put(item.key, item.value); batch.put(key, value);
total += item.value.length;
total += key.length + 80;
total += value.length + 80;
if (total >= hwm) { if (total >= hwm) {
total = 0; total = 0;
@ -705,7 +654,7 @@ Batch.prototype.del = function del(key) {
Batch.prototype.write = function write() { Batch.prototype.write = function write() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.batch.write(co.wrap(resolve, reject)); this.batch.write(wrap(resolve, reject));
}); });
}; };
@ -727,10 +676,12 @@ Batch.prototype.clear = function clear() {
*/ */
function Iterator(db, options) { function Iterator(db, options) {
options = new IteratorOptions(options); this.options = new IteratorOptions(options);
options.keyAsBuffer = db.options.bufferKeys; this.options.keyAsBuffer = db.options.bufferKeys;
this.iter = db.db.iterator(this.options);
this.iter = db.db.iterator(options); this.key = null;
this.value = null;
this.valid = true;
} }
/** /**
@ -742,16 +693,36 @@ Iterator.prototype.next = function next() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.iter.next((err, key, value) => { this.iter.next((err, key, value) => {
if (err) { if (err) {
this.iter.end(() => reject(err)); this.iter.end(() => {
this.key = null;
this.value = null;
this.valid = false;
reject(err);
});
return; return;
} }
if (key === undefined && value === undefined) { if (key === undefined && value === undefined) {
this.iter.end(co.wrap(resolve, reject)); this.iter.end((err) => {
if (err) {
reject(err);
return;
}
this.key = null;
this.value = null;
this.valid = false;
resolve(false);
});
return; return;
} }
resolve(new IteratorItem(key, value)); this.key = key;
this.value = value;
resolve(true);
}); });
}); });
}; };
@ -772,7 +743,18 @@ Iterator.prototype.seek = function seek(key) {
Iterator.prototype.end = function end() { Iterator.prototype.end = function end() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.iter.end(co.wrap(resolve, reject)); this.iter.end((err) => {
if (err) {
reject(err);
return;
}
this.key = null;
this.value = null;
this.valid = false;
resolve();
});
}); });
}; };
@ -924,6 +906,8 @@ LLUOptions.prototype.fromOptions = function fromOptions(options) {
function IteratorOptions(options) { function IteratorOptions(options) {
this.gte = null; this.gte = null;
this.lte = null; this.lte = null;
this.gt = null;
this.lt = null;
this.keys = true; this.keys = true;
this.values = false; this.values = false;
this.fillCache = false; this.fillCache = false;
@ -959,6 +943,16 @@ IteratorOptions.prototype.fromOptions = function fromOptions(options) {
this.lte = options.lte; this.lte = options.lte;
} }
if (options.gt != null) {
assert(Buffer.isBuffer(options.gt) || typeof options.gt === 'string');
this.gt = options.gt;
}
if (options.lt != null) {
assert(Buffer.isBuffer(options.lt) || typeof options.lt === 'string');
this.lt = options.lt;
}
if (options.keys != null) { if (options.keys != null) {
assert(typeof options.keys === 'boolean'); assert(typeof options.keys === 'boolean');
this.keys = options.keys; this.keys = options.keys;
@ -979,11 +973,6 @@ IteratorOptions.prototype.fromOptions = function fromOptions(options) {
this.keyAsBuffer = options.keyAsBuffer; this.keyAsBuffer = options.keyAsBuffer;
} }
if (options.valueAsBuffer != null) {
assert(typeof options.valueAsBuffer === 'boolean');
this.valueAsBuffer = options.valueAsBuffer;
}
if (options.reverse != null) { if (options.reverse != null) {
assert(typeof options.reverse === 'boolean'); assert(typeof options.reverse === 'boolean');
this.reverse = options.reverse; this.reverse = options.reverse;
@ -1014,6 +1003,16 @@ function isNotFound(err) {
|| /not\s*found/i.test(err.message); || /not\s*found/i.test(err.message);
} }
function wrap(resolve, reject) {
return function(err, result) {
if (err) {
reject(err);
return;
}
resolve(result);
};
}
VERSION_ERROR = 'Warning:' VERSION_ERROR = 'Warning:'
+ ' Your database does not match the current database version.' + ' Your database does not match the current database version.'
+ ' This is likely because the database layout or serialization' + ' This is likely because the database layout or serialization'

57
lib/wallet/walletdb.js
View File

@ -352,14 +352,11 @@ WalletDB.prototype.watch = async function watch() {
let hashes = 0; let hashes = 0;
let outpoints = 0; let outpoints = 0;
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key} = iter;
if (!item)
break;
try { try {
const data = layout.pp(item.key); const data = layout.pp(key);
this.filter.add(data, 'hex'); this.filter.add(data, 'hex');
} catch (e) { } catch (e) {
await iter.end(); await iter.end();
@ -374,14 +371,11 @@ WalletDB.prototype.watch = async function watch() {
lte: layout.o(encoding.HIGH_HASH, 0xffffffff) lte: layout.o(encoding.HIGH_HASH, 0xffffffff)
}); });
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key} = iter;
if (!item)
break;
try { try {
const [hash, index] = layout.oo(item.key); const [hash, index] = layout.oo(key);
const outpoint = new Outpoint(hash, index); const outpoint = new Outpoint(hash, index);
const data = outpoint.toRaw(); const data = outpoint.toRaw();
this.filter.add(data); this.filter.add(data);
@ -607,14 +601,11 @@ WalletDB.prototype.wipe = async function wipe() {
const batch = this.db.batch(); const batch = this.db.batch();
let total = 0; let total = 0;
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key} = iter;
if (!item)
break;
try { try {
switch (item.key[0]) { switch (key[0]) {
case 0x62: // b case 0x62: // b
case 0x63: // c case 0x63: // c
case 0x65: // e case 0x65: // e
@ -622,7 +613,7 @@ WalletDB.prototype.wipe = async function wipe() {
case 0x6f: // o case 0x6f: // o
case 0x68: // h case 0x68: // h
case 0x52: // R case 0x52: // R
batch.del(item.key); batch.del(key);
total++; total++;
break; break;
} }
@ -660,14 +651,14 @@ WalletDB.prototype.getDepth = async function getDepth() {
limit: 1 limit: 1
}); });
const item = await iter.next(); if (!await iter.next())
if (!item)
return 1; return 1;
const {key} = iter;
await iter.end(); await iter.end();
const depth = layout.ww(item.key); const depth = layout.ww(key);
return depth + 1; return depth + 1;
}; };
@ -1578,14 +1569,9 @@ WalletDB.prototype.resetState = async function resetState(tip, marked) {
values: false values: false
}); });
for (;;) { while (await iter.next()) {
const item = await iter.next();
if (!item)
break;
try { try {
batch.del(item.key); batch.del(iter.key);
} catch (e) { } catch (e) {
await iter.end(); await iter.end();
throw e; throw e;
@ -1849,15 +1835,12 @@ WalletDB.prototype.revert = async function revert(target) {
let total = 0; let total = 0;
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key, value} = iter;
if (!item)
break;
try { try {
const height = layout.bb(item.key); const height = layout.bb(key);
const block = BlockMapRecord.fromRaw(height, item.value); const block = BlockMapRecord.fromRaw(height, value);
const txs = block.toArray(); const txs = block.toArray();
total += txs.length; total += txs.length;

12
migrate/chaindb0to1.js
View File

@ -83,14 +83,10 @@ async function updateEndian() {
values: true values: true
}); });
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key, value} = iter;
batch.del(key);
if (!item) batch.put(makeKey(key), value);
break;
batch.del(item.key);
batch.put(makeKey(item.key), item.value);
total++; total++;
} }

14
migrate/chaindb1to2.js
View File

@ -120,14 +120,10 @@ async function reserializeCoins() {
values: true values: true
}); });
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key, value} = iter;
const hash = key.toString('hex', 1, 33);
if (!item) const old = OldCoins.fromRaw(value, hash);
break;
const hash = item.key.toString('hex', 1, 33);
const old = OldCoins.fromRaw(item.value, hash);
const coins = new Coins(); const coins = new Coins();
coins.version = old.version; coins.version = old.version;
@ -153,7 +149,7 @@ async function reserializeCoins() {
coins.cleanup(); coins.cleanup();
batch.put(item.key, coins.toRaw()); batch.put(key, coins.toRaw());
if (++total % 100000 === 0) if (++total % 100000 === 0)
console.log('Reserialized %d coins.', total); console.log('Reserialized %d coins.', total);

9
migrate/chaindb2to3.js
View File

@ -274,13 +274,10 @@ async function cleanupIndex() {
let batch = db.batch(); let batch = db.batch();
let total = 0; let total = 0;
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key} = iter;
if (!item) batch.del(key);
break;
batch.del(item.key);
if (++total % 10000 === 0) { if (++total % 10000 === 0) {
console.log('Cleaned up %d undo records.', total); console.log('Cleaned up %d undo records.', total);

14
migrate/walletdb2to3.js
View File

@ -61,15 +61,13 @@ async function updatePathMap() {
console.log('Migrating path map.'); console.log('Migrating path map.');
for (;;) { while (await iter.next()) {
const item = await iter.next(); const {key, value} = iter;
if (!item)
break;
total++; total++;
const hash = layout.pp(item.key);
const oldPaths = parsePaths(item.value, hash); const hash = layout.pp(key);
const oldPaths = parsePaths(value, hash);
const keys = Object.keys(oldPaths); const keys = Object.keys(oldPaths);
for (let i = 0; i < keys.length; i++) { for (let i = 0; i < keys.length; i++) {
@ -91,7 +89,7 @@ async function updatePathMap() {
batch.put(layout.P(key, hash), path.toRaw()); batch.put(layout.P(key, hash), path.toRaw());
} }
batch.put(item.key, serializeWallets(keys.sort())); batch.put(key, serializeWallets(keys.sort()));
} }
console.log('Migrated %d paths.', total); console.log('Migrated %d paths.', total);