From 9e2dd9145f10507441420369fe55e34b27086e06 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 15 Aug 2016 03:51:51 -0700 Subject: [PATCH] wallet: switch to using number ids. --- lib/bcoin/keyring.js | 20 +- lib/bcoin/protocol/constants.js | 9 + lib/bcoin/txdb.js | 150 ++++++++------- lib/bcoin/wallet.js | 68 +++++-- lib/bcoin/walletdb.js | 326 +++++++++++++++++++------------- test/wallet-test.js | 8 +- 6 files changed, 355 insertions(+), 226 deletions(-) diff --git a/lib/bcoin/keyring.js b/lib/bcoin/keyring.js index 0a24e68b..12fa178d 100644 --- a/lib/bcoin/keyring.js +++ b/lib/bcoin/keyring.js @@ -41,7 +41,8 @@ function KeyRing(options) { this.m = 1; this.n = 1; this.witness = false; - this.id = null; + this.id = 0; + this.label = null; this.name = null; this.account = 0; this.change = 0; @@ -99,10 +100,15 @@ KeyRing.prototype.fromOptions = function fromOptions(options) { } if (options.id) { - assert(utils.isAlpha(options.id)); + assert(utils.isNumber(options.id)); this.id = options.id; } + if (options.label) { + assert(utils.isAlpha(options.label)); + this.label = options.label; + } + if (options.name) { assert(utils.isAlpha(options.name)); this.name = options.name; @@ -656,6 +662,7 @@ KeyRing.prototype.toJSON = function toJSON() { n: this.n, witness: this.witness, id: this.id, + label: this.label, name: this.name, account: this.account, change: this.change, @@ -683,7 +690,8 @@ KeyRing.prototype.fromJSON = function fromJSON(json) { assert(utils.isNumber(json.m)); assert(utils.isNumber(json.n)); assert(typeof json.witness === 'boolean'); - assert(!json.id || utils.isAlpha(json.id)); + assert(!json.id || utils.isNumber(json.id)); + assert(!json.label || utils.isAlpha(json.label)); assert(!json.name || utils.isAlpha(json.name)); assert(utils.isNumber(json.account)); assert(utils.isNumber(json.change)); @@ -733,7 +741,8 @@ KeyRing.prototype.toRaw = function toRaw(writer) { p.writeU8(this.m); p.writeU8(this.n); p.writeU8(this.witness ? 1 : 0); - p.writeVarString(this.id, 'utf8'); + p.writeU32(this.id); + p.writeVarString(this.label, 'utf8'); p.writeVarString(this.name, 'utf8'); p.writeU32(this.account); p.writeU32(this.change); @@ -765,7 +774,8 @@ KeyRing.prototype.fromRaw = function fromRaw(data) { this.m = p.readU8(); this.n = p.readU8(); this.witness = p.readU8() === 1; - this.id = p.readVarString('utf8'); + this.id = p.readU32(); + this.label = p.readVarString('utf8'); this.name = p.readVarString('utf8'); this.account = p.readU32(); this.change = p.readU32(); diff --git a/lib/bcoin/protocol/constants.js b/lib/bcoin/protocol/constants.js index 48f339a5..e43a7e57 100644 --- a/lib/bcoin/protocol/constants.js +++ b/lib/bcoin/protocol/constants.js @@ -611,6 +611,15 @@ exports.MAX_HASH = new Buffer( exports.NULL_HASH = '0000000000000000000000000000000000000000000000000000000000000000'; +/** + * A hash of all 0xff. + * @const {String} + * @default + */ + +exports.HIGH_HASH = + 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + /** * A hash of all zeroes. * @const {Buffer} diff --git a/lib/bcoin/txdb.js b/lib/bcoin/txdb.js index 4b17c83d..16029329 100644 --- a/lib/bcoin/txdb.js +++ b/lib/bcoin/txdb.js @@ -27,6 +27,7 @@ var bcoin = require('./env'); var utils = require('./utils'); var assert = bcoin.utils.assert; +var constants = bcoin.protocol.constants; var DUMMY = new Buffer([0]); var pad32 = utils.pad32; var BufferReader = require('./reader'); @@ -89,7 +90,7 @@ TXDB.prototype.open = function open(callback) { TXDB.prototype.prefix = function prefix(key) { assert(this.wallet.id); - return 't/' + this.wallet.id + '/' + key; + return 't/' + pad32(this.wallet.id) + '/' + key; }; /** @@ -234,7 +235,7 @@ TXDB.prototype.commit = function commit(callback) { */ TXDB.prototype.getInfo = function getInfo(tx, callback) { - this.walletdb.getPathInfo(this.wallet.id, tx, callback); + this.walletdb.getPathInfo(this.wallet, tx, callback); }; /** @@ -415,7 +416,7 @@ TXDB.prototype._verify = function _verify(tx, info, callback) { TXDB.prototype._resolveOrphans = function _resolveOrphans(tx, index, callback) { var self = this; var hash = tx.hash('hex'); - var key = hash + '/' + index; + var key = hash + '/' + pad32(index); var coin; this._getOrphans(key, function(err, orphans) { @@ -446,7 +447,7 @@ TXDB.prototype._resolveOrphans = function _resolveOrphans(tx, index, callback) { // Verify that input script is correct, if not - add // output to unspent and remove orphan from storage if (!self.options.verify || orphan.verifyInput(input.index)) { - self.put('d/' + input.hash + '/' + input.index, coin.toRaw()); + self.put('d/' + input.hash + '/' + pad32(input.index), coin.toRaw()); return callback(null, true); } @@ -514,7 +515,7 @@ TXDB.prototype.add = function add(tx, info, callback) { self.put('m/' + pad32(tx.ps) + '/' + hash, DUMMY); for (i = 0; i < info.accounts.length; i++) { - id = info.accounts[i]; + id = pad32(info.accounts[i]); self.put('T/' + id + '/' + hash, DUMMY); if (tx.ts === 0) self.put('P/' + id + '/' + hash, DUMMY); @@ -538,7 +539,7 @@ TXDB.prototype.add = function add(tx, info, callback) { if (!path) return next(); - key = prevout.hash + '/' + prevout.index; + key = prevout.hash + '/' + pad32(prevout.index); // s/[outpoint-key] -> [spender-hash]|[spender-input-index] outpoint = bcoin.outpoint.fromTX(tx, i).toRaw(); @@ -549,8 +550,8 @@ TXDB.prototype.add = function add(tx, info, callback) { return self._addOrphan(key, outpoint, next); self.del('c/' + key); - self.del('C/' + path.account + '/' + key); - self.put('d/' + hash + '/' + i, input.coin.toRaw()); + self.del('C/' + pad32(path.account) + '/' + key); + self.put('d/' + hash + '/' + pad32(i), input.coin.toRaw()); self.balance.sub(input.coin); self.coinCache.remove(key); @@ -565,7 +566,7 @@ TXDB.prototype.add = function add(tx, info, callback) { // Add unspent outputs or resolve orphans utils.forEachSerial(tx.outputs, function(output, next, i) { var address = output.getHash('hex'); - var key = hash + '/' + i; + var key = hash + '/' + pad32(i); var coin; path = info.getPath(address); @@ -586,7 +587,7 @@ TXDB.prototype.add = function add(tx, info, callback) { coin = coin.toRaw(); self.put('c/' + key, coin); - self.put('C/' + path.account + '/' + key, DUMMY); + self.put('C/' + pad32(path.account) + '/' + key, DUMMY); self.coinCache.set(key, coin); @@ -752,7 +753,7 @@ TXDB.prototype.isDoubleSpend = function isDoubleSpend(tx, callback) { */ TXDB.prototype.isSpent = function isSpent(hash, index, callback) { - var key = 's/' + hash + '/' + index; + var key = 's/' + hash + '/' + pad32(index); this.fetch(key, function(data) { return bcoin.outpoint.fromRaw(data); }, callback); @@ -805,14 +806,14 @@ TXDB.prototype._confirm = function _confirm(tx, info, callback) { self.put('h/' + pad32(tx.height) + '/' + hash, DUMMY); for (i = 0; i < info.accounts.length; i++) { - id = info.accounts[i]; + id = pad32(info.accounts[i]); self.del('P/' + id + '/' + hash); self.put('H/' + id + '/' + pad32(tx.height) + '/' + hash, DUMMY); } utils.forEachSerial(tx.outputs, function(output, next, i) { var address = output.getHash('hex'); - var key = hash + '/' + i; + var key = hash + '/' + pad32(i); // Only update coins if this output is ours. if (!info.hasPath(address)) @@ -923,7 +924,7 @@ TXDB.prototype._remove = function remove(tx, info, callback) { this.del('m/' + pad32(tx.ps) + '/' + hash); for (i = 0; i < info.accounts.length; i++) { - id = info.accounts[i]; + id = pad32(info.accounts[i]); this.del('T/' + id + '/' + hash); if (tx.ts === 0) this.del('P/' + id + '/' + hash); @@ -938,7 +939,7 @@ TXDB.prototype._remove = function remove(tx, info, callback) { for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - key = input.prevout.hash + '/' + input.prevout.index; + key = input.prevout.hash + '/' + pad32(input.prevout.index); address = input.getHash('hex'); if (tx.isCoinbase()) @@ -957,8 +958,8 @@ TXDB.prototype._remove = function remove(tx, info, callback) { coin = input.coin.toRaw(); self.put('c/' + key, coin); - self.put('C/' + path.account + '/' + key, DUMMY); - self.del('d/' + hash + '/' + i); + self.put('C/' + pad32(path.account) + '/' + key, DUMMY); + self.del('d/' + hash + '/' + pad32(i)); self.del('s/' + key); self.del('o/' + key); @@ -967,7 +968,7 @@ TXDB.prototype._remove = function remove(tx, info, callback) { for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; - key = hash + '/' + i; + key = hash + '/' + pad32(i); address = output.getHash('hex'); path = info.getPath(address); @@ -980,7 +981,7 @@ TXDB.prototype._remove = function remove(tx, info, callback) { self.balance.sub(coin); self.del('c/' + key); - self.del('C/' + path.account + '/' + key); + self.del('C/' + pad32(path.account) + '/' + key); self.coinCache.remove(key); } @@ -1065,13 +1066,13 @@ TXDB.prototype._unconfirm = function unconfirm(tx, info, callback, force) { this.del('h/' + pad32(height) + '/' + hash); for (i = 0; i < info.accounts.length; i++) { - id = info.accounts[i]; + id = pad32(info.accounts[i]); this.put('P/' + id + '/' + hash, DUMMY); this.del('H/' + id + '/' + pad32(height) + '/' + hash); } utils.forEachSerial(tx.outputs, function(output, next, i) { - var key = hash + '/' + i; + var key = hash + '/' + pad32(i); self.getCoin(hash, i, function(err, coin) { if (err) return next(err); @@ -1114,8 +1115,12 @@ TXDB.prototype.getHistoryHashes = function getHistoryHashes(account, callback) { } this.iterate({ - gte: account != null ? 'T/' + account + '/' : 't', - lte: account != null ? 'T/' + account + '/~' : 't~', + gte: account != null + ? 'T/' + pad32(account) + '/' + constants.NULL_HASH + : 't/' + constants.NULL_HASH, + lte: account != null + ? 'T/' + pad32(account) + '/' + constants.HIGH_HASH + : 't/' + constants.HIGH_HASH, transform: function(key) { key = key.split('/'); if (account != null) @@ -1138,8 +1143,12 @@ TXDB.prototype.getUnconfirmedHashes = function getUnconfirmedHashes(account, cal } this.iterate({ - gte: account != null ? 'P/' + account + '/' : 'p', - lte: account != null ? 'P/' + account + '/~' : 'p~', + gte: account != null + ? 'P/' + pad32(account) + '/' + constants.NULL_HASH + : 'p/' + constants.NULL_HASH, + lte: account != null + ? 'P/' + pad32(account) + '/' + constants.HIGH_HASH + : 'p/' + constants.HIGH_HASH, transform: function(key) { key = key.split('/'); if (account != null) @@ -1162,8 +1171,12 @@ TXDB.prototype.getCoinHashes = function getCoinHashes(account, callback) { } this.iterate({ - gte: account != null ? 'C/' + account + '/' : 'c', - lte: account != null ? 'C/' + account + '/~' : 'c~', + gte: account != null + ? 'C/' + pad32(account) + '/' + constants.NULL_HASH + '/' + pad32(0) + : 'c/' + constants.NULL_HASH + '/' + pad32(0), + lte: account != null + ? 'C/' + pad32(account) + '/' + constants.HIGH_HASH + '/' + pad32(0xffffffff) + : 'c/' + constants.HIGH_HASH + '/' + pad32(0xffffffff), transform: function(key) { key = key.split('/'); if (account != null) @@ -1193,11 +1206,11 @@ TXDB.prototype.getHeightRangeHashes = function getHeightRangeHashes(account, opt this.iterate({ gte: account != null - ? 'H/' + account + '/' + pad32(options.start) + '/' - : 'h/' + pad32(options.start) + '/', + ? 'H/' + pad32(account) + '/' + pad32(options.start) + '/' + constants.NULL_HASH + : 'h/' + pad32(options.start) + '/' + constants.NULL_HASH, lte: account != null - ? 'H/' + account + '/' + pad32(options.end) + '/~' - : 'h/' + pad32(options.end) + '/~', + ? 'H/' + pad32(account) + '/' + pad32(options.end) + '/' + constants.HIGH_HASH + : 'h/' + pad32(options.end) + '/' + constants.HIGH_HASH, limit: options.limit, reverse: options.reverse, transform: function(key) { @@ -1238,11 +1251,11 @@ TXDB.prototype.getRangeHashes = function getRangeHashes(account, options, callba this.iterate({ gte: account != null - ? 'M/' + account + '/' + pad32(options.start) + '/' - : 'm/' + pad32(options.start) + '/', + ? 'M/' + pad32(account) + '/' + pad32(options.start) + '/' + constants.NULL_HASH + : 'm/' + pad32(options.start) + '/' + constants.NULL_HASH, lte: account != null - ? 'M/' + account + '/' + pad32(options.end) + '/~' - : 'm/' + pad32(options.end) + '/~', + ? 'M/' + pad32(account) + '/' + pad32(options.end) + '/' + constants.HIGH_HASH + : 'm/' + pad32(options.end) + '/' + constants.HIGH_HASH, limit: options.limit, reverse: options.reverse, transform: function(key) { @@ -1339,8 +1352,8 @@ TXDB.prototype.getHistory = function getHistory(account, callback) { // Fast case this.iterate({ - gte: 't', - lte: 't~', + gte: 't/' + constants.NULL_HASH, + lte: 't/' + constants.HIGH_HASH, values: true, parse: function(data) { return bcoin.tx.fromExtended(data); @@ -1483,8 +1496,8 @@ TXDB.prototype.getCoins = function getCoins(account, callback) { // Fast case this.iterate({ - gte: 'c', - lte: 'c~', + gte: 'c/' + constants.NULL_HASH + '/' + pad32(0), + lte: 'c/' + constants.HIGH_HASH + '/' + pad32(0xffffffff), keys: true, values: true, parse: function(data, key) { @@ -1494,7 +1507,7 @@ TXDB.prototype.getCoins = function getCoins(account, callback) { var coin = bcoin.coin.fromRaw(data); coin.hash = hash; coin.index = index; - key = hash + '/' + index; + key = hash + '/' + pad32(index); self.coinCache.set(key, data); return coin; } @@ -1553,8 +1566,8 @@ TXDB.prototype.fillHistory = function fillHistory(tx, callback) { hash = tx.hash('hex'); this.iterate({ - gte: 'd/' + hash + '/', - lte: 'd/' + hash + '/~', + gte: 'd/' + hash + '/' + pad32(0), + lte: 'd/' + hash + '/' + pad32(0xffffffff), keys: true, values: true, parse: function(value, key) { @@ -1704,7 +1717,7 @@ TXDB.prototype.hasTX = function hasTX(hash, callback) { TXDB.prototype.getCoin = function getCoin(hash, index, callback) { var self = this; - var key = hash + '/' + index; + var key = hash + '/' + pad32(index); var coin = this.coinCache.get(key); if (coin) { @@ -1734,7 +1747,7 @@ TXDB.prototype.getCoin = function getCoin(hash, index, callback) { */ TXDB.prototype.hasCoin = function hasCoin(hash, index, callback) { - var key = hash + '/' + index; + var key = hash + '/' + pad32(index); if (this.coinCache.has(key)) return callback(null, true); @@ -1766,11 +1779,11 @@ TXDB.prototype.getBalance = function getBalance(account, callback) { return callback(null, this.balance); // Fast case - balance = new Balance(this.wallet.id); + balance = new Balance(this.wallet); this.iterate({ - gte: 'c', - lte: 'c~', + gte: 'c/' + constants.NULL_HASH + '/' + pad32(0), + lte: 'c/' + constants.HIGH_HASH + '/' + pad32(0xffffffff), keys: true, values: true, parse: function(data, key) { @@ -1789,7 +1802,7 @@ TXDB.prototype.getBalance = function getBalance(account, callback) { else balance.confirmed += value; - key = hash + '/' + index; + key = hash + '/' + pad32(index); self.coinCache.set(key, data); } @@ -1809,7 +1822,7 @@ TXDB.prototype.getBalance = function getBalance(account, callback) { TXDB.prototype.getAccountBalance = function getBalance(account, callback) { var self = this; - var balance = new Balance(this.wallet.id); + var balance = new Balance(this.wallet); var key, coin; function parse(data) { @@ -1831,7 +1844,7 @@ TXDB.prototype.getAccountBalance = function getBalance(account, callback) { return callback(err); utils.forEachSerial(hashes, function(hash, next) { - key = hash[0] + '/' + hash[1]; + key = hash[0] + '/' + pad32(hash[1]); coin = self.coinCache.get(key); if (coin) { @@ -1934,22 +1947,23 @@ TXDB.prototype.abandon = function abandon(hash, callback, force) { * Details */ -function Details(db, id, tx, table) { - this.db = db; - this.network = db.network; - this.id = id; - this.hash = tx.hash('hex'); - this.height = tx.height; - this.block = tx.block; - this.index = tx.index; - this.confirmations = tx.getConfirmations(this.db.height); - this.fee = tx.hasCoins() ? tx.getFee() : 0; - this.ts = tx.ts; - this.ps = tx.ps; - this.tx = tx; +function Details(info) { + this.db = info.db; + this.network = info.db.network; + this.id = info.id; + this.label = info.label; + this.hash = info.tx.hash('hex'); + this.height = info.tx.height; + this.block = info.tx.block; + this.index = info.tx.index; + this.confirmations = info.tx.getConfirmations(this.db.height); + this.fee = info.tx.hasCoins() ? info.tx.getFee() : 0; + this.ts = info.tx.ts; + this.ps = info.tx.ps; + this.tx = info.tx; this.inputs = []; this.outputs = []; - this.init(table); + this.init(info.table); } Details.prototype.init = function init(table) { @@ -1980,6 +1994,7 @@ Details.prototype._insert = function _insert(vector, target, table) { for (j = 0; j < paths.length; j++) { path = paths[j]; if (path.id === this.id) { + path.label = this.label; member.path = path; break; } @@ -1994,6 +2009,7 @@ Details.prototype.toJSON = function toJSON() { var self = this; return { id: this.id, + label: this.label, hash: utils.revHex(this.hash), height: this.height, block: this.block ? utils.revHex(this.block) : null, @@ -2038,8 +2054,9 @@ DetailsMember.prototype.toJSON = function toJSON(network) { * Balance */ -function Balance(id) { - this.id = id; +function Balance(wallet) { + this.id = wallet.id; + this.label = wallet.label; this.unconfirmed = 0; this.confirmed = 0; this.total = 0; @@ -2074,6 +2091,7 @@ Balance.prototype.unconfirm = function unconfirm(value) { Balance.prototype.toJSON = function toJSON() { return { id: this.id, + label: this.label, unconfirmed: utils.btc(this.unconfirmed), confirmed: utils.btc(this.confirmed), total: utils.btc(this.total) diff --git a/lib/bcoin/wallet.js b/lib/bcoin/wallet.js index 6ed4fa8d..8a01d9d4 100644 --- a/lib/bcoin/wallet.js +++ b/lib/bcoin/wallet.js @@ -56,7 +56,8 @@ function Wallet(db, options) { this.writeLock = new bcoin.locker(this); this.fundLock = new bcoin.locker(this); - this.id = null; + this.id = 0; + this.label = null; this.master = null; this.initialized = false; this.accountDepth = 0; @@ -80,7 +81,7 @@ utils.inherits(Wallet, EventEmitter); Wallet.prototype.fromOptions = function fromOptions(options) { var master = options.master; - var id, token; + var label, token; if (!master) master = bcoin.hd.fromMnemonic(null, this.network); @@ -105,13 +106,18 @@ Wallet.prototype.fromOptions = function fromOptions(options) { this.accountDepth = options.accountDepth; } - if (options.id) { - assert(utils.isAlpha(options.id), 'Wallet ID must be alphanumeric.'); - id = options.id; + if (options.id != null) { + assert(utils.isNumber(options.id)); + this.id = options.id; } - if (!id) - id = this.getID(); + if (options.label) { + assert(utils.isAlpha(options.label), 'Wallet ID must be alphanumeric.'); + label = options.label; + } + + if (!label) + label = this.getLabel(); if (options.token) { assert(Buffer.isBuffer(options.token)); @@ -127,7 +133,7 @@ Wallet.prototype.fromOptions = function fromOptions(options) { if (!token) token = this.getToken(this.master.key, this.tokenDepth); - this.id = id; + this.label = label; this.token = token; return this; @@ -420,7 +426,7 @@ Wallet.prototype.unlock = function unlock(passphrase, timeout, callback) { * @returns {Base58String} */ -Wallet.prototype.getID = function getID() { +Wallet.prototype.getLabel = function getLabel() { var key, p, hash; assert(this.master.key, 'Cannot derive id.'); @@ -492,6 +498,7 @@ Wallet.prototype.createAccount = function createAccount(options, callback, force options = { network: self.network, id: self.id, + label: self.label, name: self.accountDepth === 0 ? 'default' : options.name, witness: options.witness, accountKey: key.hdPublicKey, @@ -570,12 +577,25 @@ Wallet.prototype.getAddresses = function getAddresses(callback) { */ Wallet.prototype.getAccount = function getAccount(account, callback) { + var self = this; + if (this.account) { if (account === 0 || account === 'default') return callback(null, this.account); } - return this.db.getAccount(this.id, account, callback); + this.db.getAccount(this.id, account, function(err, account) { + if (err) + return callback(err); + + if (!account) + return callback(); + + account.id = self.id; + account.label = self.label; + + return callback(null, account); + }); }; /** @@ -1877,6 +1897,7 @@ Wallet.prototype.__defineGetter__('changeAddress', function() { Wallet.prototype.inspect = function inspect() { return { id: this.id, + label: this.label, network: this.network.type, initialized: this.initialized, accountDepth: this.accountDepth, @@ -1898,6 +1919,7 @@ Wallet.prototype.toJSON = function toJSON() { return { network: this.network.type, id: this.id, + label: this.label, initialized: this.initialized, accountDepth: this.accountDepth, token: this.token.toString('hex'), @@ -1914,8 +1936,9 @@ Wallet.prototype.toJSON = function toJSON() { */ Wallet.prototype.fromJSON = function fromJSON(json) { - assert(utils.isAlpha(json.id), 'Wallet ID must be alphanumeric.'); + assert(utils.isNumber(json.id)); assert(typeof json.initialized === 'boolean'); + assert(utils.isAlpha(json.label), 'Wallet ID must be alphanumeric.'); assert(utils.isNumber(json.accountDepth)); assert(typeof json.token === 'string'); assert(json.token.length === 64); @@ -1923,6 +1946,7 @@ Wallet.prototype.fromJSON = function fromJSON(json) { this.network = bcoin.network.get(json.network); this.id = json.id; + this.label = json.label; this.initialized = json.initialized; this.accountDepth = json.accountDepth; this.token = new Buffer(json.token, 'hex'); @@ -1940,7 +1964,8 @@ Wallet.prototype.toRaw = function toRaw(writer) { var p = new BufferWriter(writer); p.writeU32(this.network.magic); - p.writeVarString(this.id, 'utf8'); + p.writeU32(this.id); + p.writeVarString(this.label, 'utf8'); p.writeU8(this.initialized ? 1 : 0); p.writeU32(this.accountDepth); p.writeBytes(this.token); @@ -1962,7 +1987,8 @@ Wallet.prototype.toRaw = function toRaw(writer) { Wallet.prototype.fromRaw = function fromRaw(data) { var p = new BufferReader(data); this.network = bcoin.network.fromMagic(p.readU32()); - this.id = p.readVarString('utf8'); + this.id = p.readU32(); + this.label = p.readVarString('utf8'); this.initialized = p.readU8() === 1; this.accountDepth = p.readU32(); this.token = p.readBytes(32); @@ -2043,7 +2069,8 @@ function Account(db, options) { this.receiveAddress = null; this.changeAddress = null; - this.id = null; + this.id = 0; + this.label = null; this.name = null; this.witness = this.db.options.witness; this.accountKey = null; @@ -2072,11 +2099,13 @@ Account.prototype.fromOptions = function fromOptions(options) { var i; assert(options, 'Options are required.'); - assert(utils.isAlpha(options.id), 'Wallet ID must be alphanumeric.'); + assert(utils.isNumber(options.id)); + assert(utils.isAlpha(options.label), 'Wallet ID must be alphanumeric.'); assert(bcoin.hd.isHD(options.accountKey), 'Account key is required.'); assert(utils.isNumber(options.accountIndex), 'Account index is required.'); this.id = options.id; + this.label = options.label; if (options.name != null) { assert(utils.isAlpha(options.name), 'Account name must be alphanumeric.'); @@ -2481,6 +2510,7 @@ Account.prototype.deriveAddress = function deriveAddress(change, index) { network: this.network, key: key.publicKey, id: this.id, + label: this.label, name: this.name, account: this.accountIndex, change: change, @@ -2684,7 +2714,9 @@ Account.prototype.toRaw = function toRaw(writer) { var i; p.writeU32(this.network.magic); - p.writeVarString(this.id, 'utf8'); + // NOTE: Passed in by caller. + // p.writeU32(this.id); + // p.writeVarString(this.label, 'utf8'); p.writeVarString(this.name, 'utf8'); p.writeU8(this.initialized ? 1 : 0); p.writeU8(this.type === 'pubkeyhash' ? 0 : 1); @@ -2718,7 +2750,9 @@ Account.prototype.fromRaw = function fromRaw(data) { var i, count; this.network = bcoin.network.fromMagic(p.readU32()); - this.id = p.readVarString('utf8'); + // NOTE: Passed in by caller. + // this.id = p.readU32(); + // this.label = p.readVarString('utf8'); this.name = p.readVarString('utf8'); this.initialized = p.readU8() === 1; this.type = p.readU8() === 0 ? 'pubkeyhash' : 'multisig'; diff --git a/lib/bcoin/walletdb.js b/lib/bcoin/walletdb.js index 9aaa3583..f790ef39 100644 --- a/lib/bcoin/walletdb.js +++ b/lib/bcoin/walletdb.js @@ -10,18 +10,20 @@ /* * Database Layout: * (inherits all from txdb) - * W/[address] -> id & path data + * p/[address] -> id & path data * w/[id] -> wallet + * l/[label] -> wallet id * a/[id]/[index] -> account * i/[id]/[name] -> account index * R -> tip * b/[hash] -> wallet block - * e/[hash] -> tx->wallet-id map + * t/[hash] -> tx->wallet-id map */ var bcoin = require('./env'); var AsyncObject = require('./async'); var utils = require('./utils'); +var pad32 = utils.pad32; var assert = utils.assert; var constants = bcoin.protocol.constants; var BufferReader = require('./reader'); @@ -60,11 +62,13 @@ function WalletDB(options) { this.tip = this.network.genesis.hash; this.height = 0; + this.depth = 0; // We need one read lock for `get` and `create`. // It will hold locks specific to wallet ids. - this.readLock = new ReadLock(this); - this.locker = new bcoin.locker(this); + this.readLock = new MappedLock(this); + this.writeLock = new MappedLock(this); + this.txLock = new bcoin.locker(this); this.walletCache = new bcoin.lru(10000, 1); this.accountCache = new bcoin.lru(10000, 1); @@ -131,7 +135,13 @@ WalletDB.prototype._open = function open(callback) { if (err) return callback(err); - self.loadFilter(callback); + self.getDepth(function(err, depth) { + if (err) + return callback(err); + + self.depth = depth; + self.loadFilter(callback); + }); }); }); }); @@ -160,12 +170,54 @@ WalletDB.prototype._close = function close(callback) { }; /** - * Invoke mutex lock. - * @returns {Function} unlock + * Get current wallet ID depth. + * @private + * @param {Function} callback */ -WalletDB.prototype._lock = function lock(id, func, args, force) { - return this.readLock.lock(id, func, args, force); +WalletDB.prototype.getDepth = function getDepth(callback) { + var opt, iter, parts, depth; + + // This may seem like a strange way to do + // this, but updating a global state when + // creating a new wallet is actually pretty + // damn tricky. They would be major atomicity + // issues if updating a global state inside + // a "scoped" state. So, we avoid all the + // nonsense of adding a global lock to + // walletdb.create by simply seeking to the + // highest wallet id. + iter = this.db.iterator({ + gte: 'w/' + pad32(0), + lte: 'w/' + pad32(0xffffffff), + reverse: true, + keys: true, + values: false, + fillCache: false, + keyAsBuffer: false, + valueAsBuffer: true + }); + + iter.next(function(err, key, value) { + if (err) { + return iter.end(function() { + callback(err); + }); + } + + iter.end(function(err) { + if (err) + return callback(err); + + if (key === undefined) + return callback(null, 1); + + parts = key.split('/'); + depth = +parts[1]; + + callback(null, depth + 1); + }); + }); }; /** @@ -175,7 +227,7 @@ WalletDB.prototype._lock = function lock(id, func, args, force) { */ WalletDB.prototype.start = function start(id) { - assert(utils.isAlpha(id), 'Bad ID for batch.'); + assert(utils.isNumber(id), 'Bad ID for batch.'); assert(!this.batches[id], 'Batch already started.'); this.batches[id] = this.db.batch(); }; @@ -201,7 +253,7 @@ WalletDB.prototype.drop = function drop(id) { WalletDB.prototype.batch = function batch(id) { var batch; - assert(utils.isAlpha(id), 'Bad ID for batch.'); + assert(utils.isNumber(id), 'Bad ID for batch.'); batch = this.batches[id]; assert(batch, 'Batch does not exist.'); return batch; @@ -271,8 +323,8 @@ WalletDB.prototype.testFilter = function testFilter(addresses) { WalletDB.prototype.dump = function dump(callback) { var records = {}; this.db.each({ - gte: 'w', - lte: 'w~', + gte: ' ', + lte: '~', keys: true, values: true }, function(key, value, next) { @@ -306,6 +358,33 @@ WalletDB.prototype.unregister = function unregister(wallet) { delete this.wallets[wallet.id]; }; +/** + * Map wallet label to wallet id. + * @param {String} label + * @param {Function} callback + */ + +WalletDB.prototype.getWalletID = function getWalletID(label, callback) { + var id; + + if (!label) + return callback(); + + if (typeof label === 'number') + return callback(null, label); + + id = this.walletCache.get(label); + + if (id) + return callback(null, id); + + this.db.fetch('l/' + label, function(data) { + id = data.readUInt32LE(0, true); + self.walletCache.set(label, id); + return id; + }, callback); +}; + /** * Get a wallet from the database, setup watcher. * @param {WalletID} id @@ -314,41 +393,36 @@ WalletDB.prototype.unregister = function unregister(wallet) { WalletDB.prototype.get = function get(id, callback) { var self = this; - var unlock, wallet; - unlock = this._lock(id, get, [id, callback]); - - if (!unlock) - return; - - callback = utils.wrap(callback, unlock); - - if (!id) - return callback(); - - wallet = this.wallets[id]; - - if (wallet) - return callback(null, wallet); - - this._get(id, function(err, wallet) { + this.getWalletID(id, function(err, id) { if (err) return callback(err); - if (!wallet) + if (!id) return callback(); - try { - self.register(wallet); - } catch (e) { - return callback(e); - } - - wallet.open(function(err) { + self._get(id, function(err, wallet, watched) { if (err) return callback(err); - return callback(null, wallet); + if (!wallet) + return callback(); + + if (watched) + return callback(null, wallet); + + try { + self.register(wallet); + } catch (e) { + return callback(e); + } + + wallet.open(function(err) { + if (err) + return callback(err); + + return callback(null, wallet); + }); }); }); }; @@ -362,33 +436,23 @@ WalletDB.prototype.get = function get(id, callback) { WalletDB.prototype._get = function get(id, callback) { var self = this; - var wallet; + var unlock, wallet; - if (!id) - return callback(); + unlock = this.readLock.lock(id, get, [id, callback]); - wallet = this.walletCache.get(id); + if (!unlock) + return; + + callback = utils.wrap(callback, unlock); + + wallet = this.wallets[id]; if (wallet) - return callback(null, wallet); + return callback(null, wallet, true); - this.db.get('w/' + id, function(err, data) { - if (err) - return callback(err); - - if (!data) - return callback(); - - try { - wallet = bcoin.wallet.fromRaw(self, data); - } catch (e) { - return callback(e); - } - - self.walletCache.set(id, wallet); - - return callback(null, wallet); - }); + this.db.fetch('w/' + pad32(id), function(data) { + return bcoin.wallet.fromRaw(self, data); + }, callback); }; /** @@ -399,8 +463,11 @@ WalletDB.prototype._get = function get(id, callback) { WalletDB.prototype.save = function save(wallet) { var batch = this.batch(wallet.id); - this.walletCache.set(wallet.id, wallet); - batch.put('w/' + wallet.id, wallet.toRaw()); + var id = new Buffer(4); + this.walletCache.set(wallet.label, wallet.id); + batch.put('w/' + pad32(wallet.id), wallet.toRaw()); + id.writeUInt32LE(wallet.id, 0, true); + batch.put('l/' + wallet.label, id); }; /** @@ -447,14 +514,14 @@ WalletDB.prototype.create = function create(options, callback) { options = {}; } - unlock = this._lock(options.id, create, [options, callback]); + unlock = this.writeLock.lock(options.label, create, [options, callback]); if (!unlock) return; callback = utils.wrap(callback, unlock); - this.has(options.id, function(err, exists) { + this.has(options.label, function(err, exists) { if (err) return callback(err); @@ -466,6 +533,7 @@ WalletDB.prototype.create = function create(options, callback) { try { wallet = bcoin.wallet.fromOptions(self, options); + wallet.id = self.depth++; } catch (e) { return callback(e); } @@ -480,7 +548,7 @@ WalletDB.prototype.create = function create(options, callback) { if (err) return callback(err); - self.logger.info('Created wallet %s.', wallet.id); + self.logger.info('Created wallet %s.', wallet.label); return callback(null, wallet); }); @@ -494,16 +562,11 @@ WalletDB.prototype.create = function create(options, callback) { */ WalletDB.prototype.has = function has(id, callback) { - if (!id) - return callback(null, false); - - if (this.wallets[id]) - return callback(null, true); - - if (this.walletCache.has(id)) - return callback(null, true); - - this.db.has('w/' + id, callback); + this.getWalletID(id, function(err, id) { + if (err) + return callback(err); + return callback(null, id != null); + }); }; /** @@ -516,7 +579,7 @@ WalletDB.prototype.has = function has(id, callback) { WalletDB.prototype.ensure = function ensure(options, callback) { var self = this; - this.get(options.id, function(err, wallet) { + this.get(options.label, function(err, wallet) { if (err) return callback(err); @@ -571,29 +634,17 @@ WalletDB.prototype.getAccount = function getAccount(id, name, callback) { WalletDB.prototype._getAccount = function getAccount(id, index, callback) { var self = this; - var key = id + '/' + index; + var key = pad32(id) + '/' + pad32(index); var account = this.accountCache.get(key); if (account) return callback(null, account); - this.db.get('a/' + key, function(err, data) { - if (err) - return callback(err); - - if (!data) - return callback(); - - try { - account = bcoin.account.fromRaw(self, data); - } catch (e) { - return callback(e); - } - + this.db.fetch('a/' + key, function(data) { + account = bcoin.account.fromRaw(self, data); self.accountCache.set(key, account); - - return callback(null, account); - }); + return account; + }, callback); }; /** @@ -606,12 +657,12 @@ WalletDB.prototype.getAccounts = function getAccounts(id, callback) { var map = []; var i, accounts; - if (!utils.isAlpha(id)) + if (!utils.isNumber(id)) return callback(new Error('Wallet IDs must be alphanumeric.')); this.db.iterate({ - gte: 'i/' + id + '/', - lte: 'i/' + id + '/~', + gte: 'i/' + pad32(id) + '/', + lte: 'i/' + pad32(id) + '/~', values: true, parse: function(value, key) { var name = key.split('/')[2]; @@ -651,7 +702,7 @@ WalletDB.prototype.getAccountIndex = function getAccountIndex(id, name, callback if (typeof name === 'number') return callback(null, name); - this.db.get('i/' + id + '/' + name, function(err, index) { + this.db.get('i/' + pad32(id) + '/' + name, function(err, index) { if (err) return callback(err); @@ -671,12 +722,12 @@ WalletDB.prototype.getAccountIndex = function getAccountIndex(id, name, callback WalletDB.prototype.saveAccount = function saveAccount(account) { var batch = this.batch(account.id); var index = new Buffer(4); - var key = account.id + '/' + account.accountIndex; + var key = pad32(account.id) + '/' + pad32(account.accountIndex); index.writeUInt32LE(account.accountIndex, 0, true); batch.put('a/' + key, account.toRaw()); - batch.put('i/' + account.id + '/' + account.name, index); + batch.put('i/' + pad32(account.id) + '/' + account.name, index); this.accountCache.set(key, account); }; @@ -742,7 +793,7 @@ WalletDB.prototype.hasAccount = function hasAccount(id, account, callback) { if (index === -1) return callback(null, false); - key = id + '/' + index; + key = pad32(id) + '/' + pad32(index); if (self.accountCache.has(key)) return callback(null, true); @@ -754,7 +805,7 @@ WalletDB.prototype.hasAccount = function hasAccount(id, account, callback) { /** * Save an address to the path map. * The path map exists in the form of: - * `W/[address-hash] -> {walletid1=path1, walletid2=path2, ...}` + * `p/[address-hash] -> {walletid1=path1, walletid2=path2, ...}` * @param {WalletID} id * @param {KeyRing[]} addresses * @param {Function} callback @@ -806,7 +857,7 @@ WalletDB.prototype.saveAddress = function saveAddress(id, addresses, callback) { self.pathCache.set(hash, paths); - batch.put('W/' + hash, serializePaths(paths)); + batch.put('p/' + hash, serializePaths(paths)); next(); }); @@ -831,7 +882,7 @@ WalletDB.prototype._getPaths = function _getPaths(hash, callback) { if (paths) return callback(null, paths); - this.db.fetch('W/' + hash, parsePaths, function(err, paths) { + this.db.fetch('p/' + hash, parsePaths, function(err, paths) { if (err) return callback(err); @@ -995,7 +1046,7 @@ WalletDB.prototype.mapWallets = function mapWallets(tx, callback) { * @param {Function} callback - Returns [Error, {@link PathInfo}]. */ -WalletDB.prototype.getPathInfo = function getPathInfo(id, tx, callback) { +WalletDB.prototype.getPathInfo = function getPathInfo(wallet, tx, callback) { var self = this; var addresses = tx.getHashes('hex'); var info; @@ -1007,7 +1058,8 @@ WalletDB.prototype.getPathInfo = function getPathInfo(id, tx, callback) { if (!table) return callback(); - info = new PathInfo(self, id, tx, table); + info = new PathInfo(self, wallet.id, tx, table); + info.label = wallet.label; return callback(null, info); }); @@ -1131,7 +1183,7 @@ WalletDB.prototype.writeBlock = function writeBlock(block, matches, callback) { for (i = 0; i < block.hashes.length; i++) { hash = block.hashes[i]; wallets = matches[i]; - batch.put('e/' + hash, serializeWallets(wallets)); + batch.put('t/' + hash, serializeWallets(wallets)); } } @@ -1189,7 +1241,7 @@ WalletDB.prototype.getBlock = function getBlock(hash, callback) { */ WalletDB.prototype.getWalletsByTX = function getWalletsByTX(hash, callback) { - this.db.fetch('e/' + hash, parseWallets, callback); + this.db.fetch('t/' + hash, parseWallets, callback); }; /** @@ -1202,7 +1254,7 @@ WalletDB.prototype.addBlock = function addBlock(entry, txs, callback, force) { var self = this; var i, block, matches, hash, unlock; - unlock = this.locker.lock(addBlock, [entry, txs, callback], force); + unlock = this.txLock.lock(addBlock, [entry, txs, callback], force); if (!unlock) return; @@ -1253,7 +1305,7 @@ WalletDB.prototype.removeBlock = function removeBlock(entry, callback, force) { var self = this; var unlock; - unlock = this.locker.lock(removeBlock, [entry, callback], force); + unlock = this.txLock.lock(removeBlock, [entry, callback], force); if (!unlock) return; @@ -1342,6 +1394,8 @@ WalletDB.prototype.addTX = function addTX(tx, callback, force) { self.logger.debug('Adding tx to wallet: %s', info.id); + info.label = wallet.label; + wallet.tx.add(tx, info, function(err) { if (err) return next(err); @@ -1397,7 +1451,9 @@ function Path() { this.account = 0; this.change = 0; this.index = 0; - this.address = null; + + // NOTE: Passed in by caller. + this.label = null; } /** @@ -1408,7 +1464,7 @@ function Path() { Path.prototype.fromRaw = function fromRaw(data) { var p = new BufferReader(data); - this.id = p.readVarString('utf8'); + this.id = p.readU32(); this.name = p.readVarString('utf8'); this.account = p.readU32(); this.change = p.readU32(); @@ -1434,7 +1490,7 @@ Path.fromRaw = function fromRaw(data) { Path.prototype.toRaw = function toRaw(writer) { var p = new BufferWriter(writer); - p.writeVarString(this.id, 'utf8'); + p.writeU32(this.id); p.writeVarString(this.name, 'utf8'); p.writeU32(this.account); p.writeU32(this.change); @@ -1455,6 +1511,7 @@ Path.prototype.toRaw = function toRaw(writer) { Path.prototype.fromKeyRing = function fromKeyRing(address) { this.id = address.id; + this.label = address.label; this.name = address.name; this.account = address.account; this.change = address.change; @@ -1492,6 +1549,7 @@ Path.prototype.toPath = function() { Path.prototype.toJSON = function toJSON() { return { id: this.id, + label: this.label, name: this.name, change: this.change === 1, path: this.toPath() @@ -1512,6 +1570,7 @@ Path.prototype.fromJSON = function fromJSON(json) { indexes[0] -= constants.hd.HARDENED; this.id = json.id; + this.label = json.label; this.name = json.name; this.account = indexes[0]; this.change = indexes[1]; @@ -1530,22 +1589,14 @@ Path.fromJSON = function fromJSON(json) { return new Path().fromJSON(json); }; -/** - * Convert path to a key in the form of (id|account). - * @returns {String} - */ - -Path.prototype.toKey = function toKey() { - return this.id + '/' + this.account; -}; - /** * Inspect the path. * @returns {String} */ Path.prototype.inspect = function() { - return ''; @@ -1572,6 +1623,9 @@ function PathInfo(db, id, tx, table) { // Wallet ID this.id = id; + // Wallet Label (passed in by caller). + this.label = null; + // Map of address hashes->paths (for everything). this.table = null; @@ -1610,7 +1664,7 @@ PathInfo.map = function map(db, tx, table) { return; for (i = 0; i < wallets.length; i++) { - id = wallets[i]; + id = +wallets[i]; info.push(new PathInfo(db, id, tx, table)); } @@ -1692,7 +1746,7 @@ PathInfo.prototype.toDetails = function toDetails() { var details = this._details; if (!details) { - details = new TXDB.Details(this.db, this.id, this.tx, this.table); + details = new TXDB.Details(this); this._details = details; } @@ -1747,7 +1801,7 @@ function serializeWallets(wallets) { for (i = 0; i < wallets.length; i++) { info = wallets[i]; - p.writeVarBytes(info.id, 'ascii'); + p.writeU32(info.id); } return p.render(); @@ -1757,7 +1811,7 @@ function parseWallets(data) { var p = new BufferReader(data); var wallets = []; while (p.left()) - wallets.push(p.readVarString('ascii')); + wallets.push(p.readU32()); return wallets; } @@ -1843,33 +1897,33 @@ WalletBlock.prototype.toJSON = function toJSON() { }; }; -function ReadLock(parent) { - if (!(this instanceof ReadLock)) - return new ReadLock(parent); +function MappedLock(parent) { + if (!(this instanceof MappedLock)) + return new MappedLock(parent); this.parent = parent; this.jobs = []; this.busy = {}; } -ReadLock.prototype.lock = function lock(id, func, args, force) { +MappedLock.prototype.lock = function lock(key, func, args, force) { var self = this; var called; - if (force || !id) { - assert(!id || this.busy[id]); + if (force || key == null) { + assert(key == null || this.busy[key]); return function unlock() { assert(!called); called = true; }; } - if (this.busy[id]) { + if (this.busy[key]) { this.jobs.push([func, args]); return; } - this.busy[id] = true; + this.busy[key] = true; return function unlock() { var item; @@ -1877,7 +1931,7 @@ ReadLock.prototype.lock = function lock(id, func, args, force) { assert(!called); called = true; - delete self.busy[id]; + delete self.busy[key]; if (self.jobs.length === 0) return; diff --git a/test/wallet-test.js b/test/wallet-test.js index f0ebeec4..27ecb610 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -1001,7 +1001,11 @@ describe('Wallet', function() { }); it('should cleanup', function(cb) { - constants.tx.COINBASE_MATURITY = 100; - cb(); + walletdb.dump(function(err, records) { + assert.ifError(err); + // utils.log(JSON.stringify(Object.keys(records), null, 2)); + constants.tx.COINBASE_MATURITY = 100; + cb(); + }); }); });