diff --git a/migrate/chaindb4to5.js b/migrate/chaindb4to6.js similarity index 54% rename from migrate/chaindb4to5.js rename to migrate/chaindb4to6.js index b9eb1f0b..0e9e8b36 100644 --- a/migrate/chaindb4to5.js +++ b/migrate/chaindb4to6.js @@ -8,8 +8,9 @@ const {resolve} = require('path'); assert(process.argv.length > 2, 'Please pass in a database path.'); -// migration - -// chaindb: leveldb to flat files +// Changes: +// 1. Moves blocks and undo blocks from leveldb to flat files. +// 2. Removes tx and addr indexes from chaindb. const db = bdb.create({ location: process.argv[2], @@ -25,29 +26,33 @@ const blockStore = new FileBlockStore({ location: location }); -async function updateVersion() { - const ver = await checkVersion(); +async function getVersion() { + const data = await db.get(layout.V.encode()); + assert(data, 'No version.'); - console.log('Updating version to %d.', ver + 1); + return data.readUInt32LE(5, true); +} + +async function updateVersion(version) { + await checkVersion(version - 1); + + console.log('Updating version to %d.', version); const buf = Buffer.allocUnsafe(5 + 4); buf.write('chain', 0, 'ascii'); - buf.writeUInt32LE(5, 5, true); + buf.writeUInt32LE(version, 5, true); const parent = db.batch(); parent.put(layout.V.encode(), buf); await parent.write(); } -async function checkVersion() { +async function checkVersion(version) { console.log('Checking version.'); - const data = await db.get(layout.V.encode()); - assert(data, 'No version.'); + const ver = await getVersion(); - const ver = data.readUInt32LE(5, true); - - if (ver !== 4) + if (ver !== version) throw Error(`DB is version ${ver}.`); return ver; @@ -113,26 +118,85 @@ async function migrateBlocks() { await parent.write(); } +async function removeKey(name, key) { + const iter = db.iterator({ + gte: key.min(), + lte: key.max(), + reverse: true, + keys: true + }); + + let batch = db.batch(); + let total = 0; + + while (await iter.next()) { + const {key} = iter; + batch.del(key); + + if (++total % 10000 === 0) { + console.log('Cleaned up %d %s index records.', total, name); + await batch.write(); + batch = db.batch(); + } + } + await batch.write(); + + console.log('Cleaned up %d %s index records.', total, name); +} + +async function migrateIndexes() { + const t = bdb.key('t', ['hash256']); + const T = bdb.key('T', ['hash', 'hash256']); + const C = bdb.key('C', ['hash', 'hash256', 'uint32']); + + await removeKey('hash -> tx', t); + await removeKey('addr -> tx', T); + await removeKey('addr -> coin', C); +} + /* * Execute */ (async () => { await db.open(); - await blockStore.ensure(); - await blockStore.open(); console.log('Opened %s.', process.argv[2]); - await checkVersion(); - await migrateBlocks(); - await migrateUndoBlocks(); - await updateVersion(); + const version = await getVersion(); + let compact = false; + + switch (version) { + case 4: + // Upgrade from version 4 to 5. + await checkVersion(4); + await blockStore.ensure(); + await blockStore.open(); + await migrateBlocks(); + await migrateUndoBlocks(); + await updateVersion(5); + await blockStore.close(); + compact = true; + case 5: + // Upgrade from version 5 to 6. + await checkVersion(5); + await migrateIndexes(); + await updateVersion(6); + compact = true; + break; + case 6: + console.log('Already upgraded.'); + break; + default: + console.log(`DB version is ${version}.`); + } + + if (compact) { + console.log('Compacting database'); + await db.compactRange(); + } - console.log('Compacting database'); - await db.compactRange(); await db.close(); - await blockStore.close(); })().then(() => { console.log('Migration complete.'); process.exit(0); diff --git a/migrate/chaindb5to6.js b/migrate/chaindb5to6.js deleted file mode 100644 index 8a85b607..00000000 --- a/migrate/chaindb5to6.js +++ /dev/null @@ -1,105 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const bdb = require('bdb'); -const layout = require('../lib/blockchain/layout'); - -// changes: -// removes tx, addr indexes i.e layout.t, layout.T, layout.C - -assert(process.argv.length > 2, 'Please pass in a database path.'); - -const db = bdb.create({ - location: process.argv[2], - memory: false, - compression: true, - cacheSize: 32 << 20, - createIfMissing: false -}); - -async function updateVersion() { - const ver = await checkVersion(); - - console.log('Updating version to %d.', ver + 1); - - const buf = Buffer.allocUnsafe(5 + 4); - buf.write('chain', 0, 'ascii'); - buf.writeUInt32LE(6, 5, true); - - const parent = db.batch(); - parent.put(layout.V.encode(), buf); - await parent.write(); -} - -async function checkVersion() { - console.log('Checking version.'); - - const data = await db.get(layout.V.encode()); - assert(data, 'No version.'); - - const ver = data.readUInt32LE(5, true); - - if (ver !== 5) - throw Error(`DB is version ${ver}.`); - - return ver; -} - -async function removeKey(name, key) { - const iter = db.iterator({ - gte: key.min(), - lte: key.max(), - reverse: true, - keys: true - }); - - let batch = db.batch(); - let total = 0; - - while (await iter.next()) { - const {key} = iter; - batch.del(key); - - if (++total % 10000 === 0) { - console.log('Cleaned up %d %s index records.', total, name); - await batch.write(); - batch = db.batch(); - } - } - await batch.write(); - - console.log('Cleaned up %d %s index records.', total, name); -} - -async function migrateIndexes() { - const t = bdb.key('t', ['hash256']); - const T = bdb.key('T', ['hash', 'hash256']); - const C = bdb.key('C', ['hash', 'hash256', 'uint32']); - - await removeKey('hash -> tx', t); - await removeKey('addr -> tx', T); - await removeKey('addr -> coin', C); -} - -/* - * Execute - */ - -(async () => { - await db.open(); - - console.log('Opened %s.', process.argv[2]); - - await checkVersion(); - await migrateIndexes(); - await updateVersion(); - - await db.compactRange(); - await db.close(); -})().then(() => { - console.log('Migration complete.'); - process.exit(0); -}).catch((err) => { - console.error(err.stack); - process.exit(1); -}); diff --git a/migrate/latest b/migrate/latest index dd0e1c13..b4ebd981 100755 --- a/migrate/latest +++ b/migrate/latest @@ -3,7 +3,8 @@ 'use strict'; const cp = require('child_process'); -const res = require('path').resolve; +const fs = require('bfile'); +const {resolve} = require('path'); const {argv} = process; if (argv.length < 3) { @@ -31,8 +32,16 @@ function exec(file, ...args) { const node = argv[0]; const prefix = argv[2]; -exec(node, res(__dirname, 'chaindb3to4.js'), res(prefix, 'chain')); -exec(node, res(__dirname, 'chaindb3to4.js'), res(prefix, 'spvchain')); -exec(node, res(__dirname, 'chaindb4to5.js'), res(prefix, 'chain')); -exec(node, res(__dirname, 'chaindb5to6.js'), res(prefix, 'chain')); -exec(node, res(__dirname, 'walletdb6to7.js'), res(prefix, 'wallet')); +const chain = resolve(prefix, 'chain'); +const spvchain = resolve(prefix, 'spvchain'); + +(async () => { + if (await fs.exists(chain)) + exec(node, resolve(__dirname, 'chaindb4to6.js'), chain); + + if (await fs.exists(spvchain)) + exec(node, resolve(__dirname, 'chaindb4to6.js'), spvchain); +})().catch((err) => { + console.error(err.stack); + process.exit(1); +});