diff --git a/lib/http/server.js b/lib/http/server.js index a5e3bd4f..9c102b2b 100644 --- a/lib/http/server.js +++ b/lib/http/server.js @@ -1574,25 +1574,6 @@ ClientSocket.prototype.testFilter = function testFilter(tx) { ClientSocket.prototype.scan = co(function* scan(start) { var scanner = this.scanner.bind(this); - var entry; - - if (this.chain.db.options.spv) { - entry = yield this.chain.db.get(start); - - if (!entry) - throw new Error('Block not found.'); - - if (!entry.isGenesis()) - start = entry.prevBlock; - - yield this.chain.reset(start); - - return; - } - - if (this.chain.db.options.prune) - throw new Error('Cannot scan in pruned mode.'); - yield this.chain.db.scan(start, this.filter, scanner); }); diff --git a/lib/node/config.js b/lib/node/config.js index 4c1fb710..518e4d85 100644 --- a/lib/node/config.js +++ b/lib/node/config.js @@ -162,7 +162,6 @@ config.parseData = function parseData(data, prefix, dirname) { options.coinCache = bool(data.coincache); options.indexTX = bool(data.indextx); options.indexAddress = bool(data.indexaddress); - options.noScan = bool(data.noscan); // Mempool options.limitFree = bool(data.limitfree); @@ -191,7 +190,6 @@ config.parseData = function parseData(data, prefix, dirname) { // Miner options.payoutAddress = str(data.payoutaddress); options.coinbaseFlags = str(data.coinbaseflags); - options.parallel = bool(data.parallel); // HTTP options.sslCert = file(data.sslcert, prefix, dirname); @@ -203,6 +201,9 @@ config.parseData = function parseData(data, prefix, dirname) { options.walletAuth = bool(data.walletauth); options.noAuth = bool(data.noauth); + // Wallet + options.wipeNoReally = bool(data.wipenoreally); + options.data = data; if (options.knownPeers != null) diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 7c67b20a..8dd9b953 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -148,6 +148,7 @@ function Fullnode(options) { witness: this.options.witness, useCheckpoints: this.options.useCheckpoints, maxFiles: this.options.maxFiles, + wipeNoReally: this.options.wipeNoReally, resolution: false, verify: false }); @@ -288,12 +289,6 @@ Fullnode.prototype._close = co(function* close() { */ Fullnode.prototype.rescan = function rescan() { - if (this.options.noScan) { - return this.walletdb.setTip( - this.chain.tip.hash, - this.chain.height); - } - // Always rescan to make sure we didn't // miss anything: there is no atomicity // between the chaindb and walletdb. diff --git a/lib/node/spvnode.js b/lib/node/spvnode.js index cf5c2e15..026fe62b 100644 --- a/lib/node/spvnode.js +++ b/lib/node/spvnode.js @@ -86,6 +86,7 @@ function SPVNode(options) { location: this.location('walletdb'), witness: this.options.witness, maxFiles: this.options.maxFiles, + wipeNoReally: this.options.wipeNoReally, resolution: true, verify: true }); @@ -137,11 +138,18 @@ SPVNode.prototype._init = function _init() { self.walletdb.addTX(tx).catch(onError); }); - this.chain.on('block', function(block, entry) { + this.chain.on('block', function(block) { self.emit('block', block); + }); + + this.chain.on('connect', function(entry, block) { self.walletdb.addBlock(entry, block.txs).catch(onError); }); + this.chain.on('disconnect', function(entry, block) { + self.walletdb.removeBlock(entry).catch(onError); + }); + this.walletdb.on('path', function(path) { self.pool.watch(path.hash, 'hex'); }); @@ -223,12 +231,6 @@ SPVNode.prototype.openFilter = co(function* openFilter() { */ SPVNode.prototype.rescan = function rescan() { - if (this.options.noScan) { - return this.walletdb.setTip( - this.chain.tip.hash, - this.chain.height); - } - // Always replay the last block to make // sure we didn't miss anything: there // is no atomicity between the chaindb @@ -244,19 +246,7 @@ SPVNode.prototype.rescan = function rescan() { */ SPVNode.prototype.scan = co(function* scan(height) { - if (height == null) - height = this.walletdb.height; - - if (typeof height === 'string') { - height = yield this.chain.db.getHeight(height); - if (height === -1) - return; - } - - if (height === 0) - return; - - yield this.chain.reset(height - 1); + return this.walletdb.rescan(this.chain.db, height); }); /** diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index b87ee3f7..4c34d08f 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -862,6 +862,81 @@ TXDB.prototype.isSpent = co(function* isSpent(hash, index) { return data != null; }); +TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) { + var hash = tx.hash('hex'); + var wids = yield this.walletdb.getWalletsByTX(hash); + var result; + + if (!wids) + wids = []; + + result = utils.binaryInsert(wids, this.wallet.wid, cmp, true); + + if (result === -1) + return; + + this.walletdb.writeTX(this.wallet, hash, wids); +}); + +TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) { + var hash = tx.hash('hex'); + var wids = yield this.walletdb.getWalletsByTX(hash); + var result, block; + + if (!wids) + return; + + result = utils.binaryRemove(wids, this.wallet.wid, cmp); + + if (!result) + return; + + if (tx.height !== -1) { + block = yield this.walletdb.getBlock(tx.height); + + if (block.remove(hash, this.wallet.wid)) + this.walletdb.writeBlock(this.wallet, block); + } + + if (wids.length === 0) { + this.walletdb.unwriteTX(this.wallet, hash); + return; + } + + this.walletdb.writeTX(this.wallet, hash, wids); +}); + +TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, record) { + var hash = tx.hash('hex'); + var block = yield this.walletdb.getBlock(record.height); + + if (!block) + block = record.clone(); + + if (!block.add(hash, this.wallet.wid)) + return; + + this.walletdb.writeBlock(this.wallet, block); +}); + +TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) { + var hash = tx.hash('hex'); + var block = yield this.walletdb.getBlock(height); + + if (!block) + return; + + if (!block.remove(hash, this.wallet.wid)) + return; + + if (block.txs.length === 0) { + this.walletdb.unwriteBlock(this.wallet, block); + return; + } + + this.walletdb.writeBlock(this.wallet, block); +}); + /** * Add transaction, potentially runs * `confirm()` and `removeConflicts()`. @@ -901,23 +976,19 @@ TXDB.prototype._add = co(function* add(tx, block) { if (existing) { // Existing tx is already confirmed. Ignore. - if (existing.height !== -1) { - if (block) { - // XXX need to return true here, could be a rescan. - } + if (existing.height !== -1) return; - } // The incoming tx won't confirm the // existing one anyway. Ignore. if (tx.height === -1) return; - // Save the original time. - tx.ps = existing.ps; + // Save the index (can't get this elsewhere). + existing.index = tx.index; // Confirm transaction. - return yield this._confirm(tx, block); + return yield this._confirm(existing, block); } if (tx.height === -1) { @@ -1118,81 +1189,6 @@ TXDB.prototype.insert = co(function* insert(tx, block) { return details; }); -TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) { - var hash = tx.hash('hex'); - var wids = yield this.walletdb.getWalletsByTX(hash); - var result; - - if (!wids) - wids = []; - - result = utils.binaryInsert(wids, this.wallet.wid, cmp, true); - - if (result === -1) - return; - - this.walletdb.writeTX(this.wallet, hash, wids); -}); - -TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) { - var hash = tx.hash('hex'); - var wids = yield this.walletdb.getWalletsByTX(hash); - var result; - - if (!wids) - return; - - result = utils.binaryRemove(wids, this.wallet.wid, cmp); - - if (!result) - return; - - yield this.removeBlockRecord(tx, tx.height); - - if (wids.length === 0) { - this.walletdb.unwriteTX(this.wallet, hash); - return; - } - - this.walletdb.writeTX(this.wallet, hash, wids); -}); - -TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, record) { - var hash = tx.hash('hex'); - var block = yield this.walletdb.getBlock(record.height); - - if (!block) - block = record.clone(); - - if (!block.add(hash, this.wallet.wid)) - return; - - this.walletdb.writeBlock(this.wallet, block); -}); - -TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) { - var hash = tx.hash('hex'); - var block = yield this.walletdb.getBlock(height); - - if (!block) - return; - - if (!block.remove(hash, this.wallet.wid)) - return; - - if (block.txs.length === 0) { - this.walletdb.unwriteBlock(this.wallet, block); - return; - } - - this.walletdb.writeBlock(this.wallet, block); -}); - - -function cmp(a, b) { - return a - b; -} - /** * Attempt to confirm a transaction. * @private @@ -1207,11 +1203,6 @@ TXDB.prototype.confirm = co(function* confirm(hash, block) { if (!tx) return; - tx.height = block.height; - tx.block = block.hash; - tx.index = -1; - tx.ts = utils.now(); - this.start(); try { @@ -1240,7 +1231,12 @@ TXDB.prototype._confirm = co(function* confirm(tx, block) { var i, account, output, coin, input, prevout; var path, credit, credits; - assert(tx.height !== -1); + if (!block) + block = this.wallet.db.tip; + + tx.height = block.height; + tx.block = block.hash; + tx.ts = block.ts; if (!tx.isCoinbase()) { credits = yield this.getSpentCredits(tx); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 3f9f246d..4fe60557 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -152,8 +152,8 @@ function WalletDB(options) { this.fees = options.fees; this.logger = options.logger || Logger.global; - this.tip = this.network.genesis.hash; - this.height = 0; + this.tip = null; + this.height = -1; this.depth = 0; this.wallets = {}; @@ -217,6 +217,9 @@ WalletDB.prototype._open = co(function* open() { yield this.db.checkVersion('V', 4); yield this.writeGenesis(); + if (this.options.wipeNoReally) + yield this.wipe(); + this.depth = yield this.getDepth(); this.logger.info( @@ -255,6 +258,55 @@ WalletDB.prototype.backup = function backup(path) { return this.db.backup(path); }; +/** + * Wipe the txdb - NEVER USE. + * @returns {Promise} + */ + +WalletDB.prototype.wipe = co(function* wipe() { + var batch = this.db.batch(); + var dummy = new Buffer(0); + var i, keys, key; + + this.logger.warning('Wiping txdb...'); + this.logger.warning('I hope you know what you\'re doing.'); + + keys = yield this.db.keys({ + gte: TXDB.layout.prefix(0x00000000, dummy), + lte: TXDB.layout.prefix(0xffffffff, dummy) + }); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + batch.del(key); + } + + keys = yield this.db.keys({ + gte: layout.b(constants.NULL_HASH), + lte: layout.b(constants.HIGH_HASH) + }); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + batch.del(key); + } + + keys = yield this.db.keys({ + gte: layout.e(constants.NULL_HASH), + lte: layout.e(constants.HIGH_HASH) + }); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + batch.del(key); + } + + batch.del(layout.R); + + yield batch.write(); + yield this.writeGenesis(); +}); + /** * Get current wallet wid depth. * @private @@ -1104,7 +1156,7 @@ WalletDB.prototype._rescan = co(function* rescan(chaindb, height) { this.logger.info('Scanning for %d addresses.', hashes.length); - yield chaindb.scan(this.tip, hashes, function(block, txs) { + yield chaindb.scan(this.tip.hash, hashes, function(block, txs) { return self._addBlock(block, txs); }); }); @@ -1317,12 +1369,15 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) { WalletDB.prototype.writeGenesis = co(function* writeGenesis() { var block = yield this.getTip(); + var genesis = this.network.genesis; + if (block) { - this.tip = block.hash; + this.tip = block; this.height = block.height; return; } - yield this.setTip(this.network.genesis.hash, 0); + + yield this.setTip(genesis.hash, 0, genesis.ts); }); /** @@ -1347,8 +1402,8 @@ WalletDB.prototype.getTip = co(function* getTip() { * @returns {Promise} */ -WalletDB.prototype.setTip = function setTip(hash, height) { - var block = new WalletBlock(hash, height); +WalletDB.prototype.setTip = function setTip(hash, height, ts) { + var block = new WalletBlock(hash, height, ts); return this.setBlock(block); }; @@ -1368,7 +1423,7 @@ WalletDB.prototype.setBlock = co(function* setBlock(block) { yield batch.write(); - this.tip = block.hash; + this.tip = block; this.height = block.height; }); @@ -1481,6 +1536,15 @@ WalletDB.prototype.addBlock = co(function* addBlock(entry, txs) { var unlock = yield this.txLock.lock(); try { yield this.rollback(entry.height - 1); + + if (entry.height <= this.height) { + this.logger.warning('Node is connecting low blocks in wallet.'); + return; + } + + if (entry.height !== this.height + 1) + throw new Error('Bad connection (height mismatch).'); + return yield this._addBlock(entry, txs); } finally { unlock(); @@ -1523,7 +1587,7 @@ WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) { } this.height = block.height; - this.tip = block.hash; + this.tip = block; this.logger.info('Connected block %s (tx=%d).', utils.revHex(block.hash), total); @@ -1542,6 +1606,15 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) { var unlock = yield this.txLock.lock(); try { yield this.rollback(entry.height); + + if (entry.height > this.height) { + this.logger.warning('Node is disconnecting high blocks in wallet.'); + return; + } + + if (entry.height !== this.height) + throw new Error('Bad disconnection (height mismatch).'); + return yield this._removeBlock(entry); } finally { unlock(); @@ -1562,8 +1635,6 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { if (!block) return; - assert(block.height > 0); - prev = yield this.getBlock(entry.height - 1); assert(prev); @@ -1578,7 +1649,7 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { } this.height = prev.height; - this.tip = prev.hash; + this.tip = prev; this.logger.warning('Disconnected block %s (tx=%d).', utils.revHex(block.hash), block.txs.length); @@ -1645,25 +1716,6 @@ WalletDB.prototype._insertTX = co(function* insertTX(tx, block) { return wids; }); -/** - * Confirm a transaction from all - * relevant wallets without a lock. - * @private - * @param {TXHash} hash - * @returns {Promise} - */ - -WalletDB.prototype._confirmTX = co(function* confirmTX(tx, block) { - var i, wid, wallet; - - for (i = 0; i < tx.wids.length; i++) { - wid = tx.wids[i]; - wallet = yield this.get(wid); - assert(wallet); - yield wallet.confirm(tx.hash, block); - } -}); - /** * Unconfirm a transaction from all * relevant wallets without a lock. @@ -1724,12 +1776,13 @@ WalletDB.prototype._zap = co(function* zap(age) { * @param {Number} height */ -function WalletBlock(hash, height) { +function WalletBlock(hash, height, ts) { if (!(this instanceof WalletBlock)) - return new WalletBlock(hash, height); + return new WalletBlock(hash, height, ts); this.hash = hash || constants.NULL_HASH; this.height = height != null ? height : -1; + this.ts = ts || 0; this.txs = []; this.index = {}; } @@ -1740,7 +1793,7 @@ function WalletBlock(hash, height) { */ WalletBlock.prototype.clone = function clone() { - return new WalletBlock(this.hash, this.height); + return new WalletBlock(this.hash, this.height, this.ts); }; /** @@ -1752,6 +1805,7 @@ WalletBlock.prototype.clone = function clone() { WalletBlock.prototype.fromEntry = function fromEntry(entry) { this.hash = entry.hash; this.height = entry.height; + this.ts = entry.ts; return this; }; @@ -1764,6 +1818,7 @@ WalletBlock.prototype.fromEntry = function fromEntry(entry) { WalletBlock.prototype.fromJSON = function fromJSON(json) { this.hash = utils.revHex(json.hash); this.height = json.height; + this.ts = json.ts; return this; }; @@ -1780,6 +1835,7 @@ WalletBlock.prototype.fromRaw = function fromRaw(data) { this.hash = p.readHash('hex'); this.height = p.readU32(); + this.ts = p.readU32(); while (p.left()) { hash = p.readHash('hex'); @@ -1804,6 +1860,7 @@ WalletBlock.prototype.fromTip = function fromTip(data) { var p = new BufferReader(data); this.hash = p.readHash('hex'); this.height = p.readU32(); + this.ts = p.readU32(); return this; }; @@ -1858,6 +1915,7 @@ WalletBlock.prototype.toTip = function toTip(writer) { p.writeHash(this.hash); p.writeU32(this.height); + p.writeU32(this.ts); if (!writer) p = p.render(); @@ -1877,6 +1935,7 @@ WalletBlock.prototype.toRaw = function toRaw(writer) { p.writeHash(this.hash); p.writeU32(this.height); + p.writeU32(this.ts); for (i = 0; i < this.txs.length; i++) { tx = this.txs[i]; @@ -1901,7 +1960,6 @@ WalletBlock.prototype.toRaw = function toRaw(writer) { WalletBlock.prototype.add = function add(hash, wid) { var tx = this.index[hash]; - var index; if (!tx) { tx = new TXHash(hash); @@ -1911,12 +1969,7 @@ WalletBlock.prototype.add = function add(hash, wid) { return true; } - index = utils.binaryInsert(tx.wids, wid, cmp, true); - - if (index === -1) - return false; - - return true; + return tx.add(wid); }; /** @@ -1933,9 +1986,7 @@ WalletBlock.prototype.remove = function remove(hash, wid) { if (!tx) return false; - result = utils.binaryRemove(tx.wids, wid, cmp); - - if (!result) + if (!tx.remove(wid)) return false; if (tx.wids.length === 0) { @@ -1971,6 +2022,26 @@ function TXHash(hash, wids) { TXHash.id = 0; +TXHash.prototype.add = function add(wid) { + return utils.binaryInsert(this.wids, wid, cmp, true) !== -1; +}; + +TXHash.prototype.remove = function remove(wid) { + return utils.binaryRemove(this.wids, wid, cmp); +}; + +TXHash.prototype.toRaw = function toRaw(writer) { + return serializeWallets(this.wids, writer); +}; + +TXHash.prototype.fromRaw = function fromRaw(data) { + return parseWallets(data); +}; + +TXHash.fromRaw = function fromRaw(hash, data) { + return new TXHash(hash).fromRaw(data); +}; + function parseWallets(data) { var p = new BufferReader(data); var wids = []; @@ -1981,8 +2052,8 @@ function parseWallets(data) { return wids; } -function serializeWallets(wids) { - var p = new BufferWriter(); +function serializeWallets(wids, writer) { + var p = new BufferWriter(writer); var i, wid; for (i = 0; i < wids.length; i++) { @@ -1990,7 +2061,10 @@ function serializeWallets(wids) { p.writeU32(wid); } - return p.render(); + if (!writer) + p = p.render(); + + return p; } function cmp(a, b) { diff --git a/test/chain-test.js b/test/chain-test.js index a6f77ec1..f454f5a3 100644 --- a/test/chain-test.js +++ b/test/chain-test.js @@ -227,7 +227,7 @@ describe('Chain', function() { assert(wallet.account.changeDepth >= 7); assert.equal(walletdb.height, chain.height); - assert.equal(walletdb.tip, chain.tip.hash); + assert.equal(walletdb.tip.hash, chain.tip.hash); txs = yield wallet.getHistory(); assert.equal(txs.length, 44);