migrate: walletdb 2 to 3.

lib/primitives/keyring.js

@@ -876,9 +876,15 @@ KeyRing.fromJSON = function fromJSON(json) {
 
 KeyRing.prototype.toRaw = function toRaw(writer) {
   var p = new BufferWriter(writer);
+  var field = 0;
 
-  p.writeU8(this.witness ? 1 : 0);
-  p.writeU8(this.nested ? 1 : 0);
+  if (this.witness)
+    field |= 1;
+
+  if (this.nested)
+    field |= 2;
+
+  p.writeU8(field);
 
   if (this.privateKey) {
     p.writeVarBytes(this.privateKey);
@@ -906,11 +912,14 @@ KeyRing.prototype.toRaw = function toRaw(writer) {
 
 KeyRing.prototype.fromRaw = function fromRaw(data, network) {
   var p = new BufferReader(data);
-  var compressed, key, script;
+  var field, compressed, key, script;
 
   this.network = Network.get(network);
-  this.witness = p.readU8() === 1;
-  this.nested = p.readU8() === 1;
+
+  field = p.readU8();
+
+  this.witness = (field & 1) !== 0;
+  this.nested = (field & 2) !== 0;
 
   key = p.readVarBytes();
 

lib/wallet/path.js

@@ -24,9 +24,9 @@ var Script = require('../script/script');
  * @property {Address|null} address
  */
 
-function Path() {
+function Path(options) {
   if (!(this instanceof Path))
-    return new Path();
+    return new Path(options);
 
   this.keyType = Path.types.HD;
 
@@ -44,6 +44,9 @@ function Path() {
   this.type = Script.types.PUBKEYHASH;
   this.version = -1;
   this.hash = null; // Passed in by caller.
+
+  if (options)
+    this.fromOptions(options);
 }
 
 /**
@@ -58,6 +61,43 @@ Path.types = {
   ADDRESS: 2
 };
 
+/**
+ * Instantiate path from options object.
+ * @private
+ * @param {Object} options
+ * @returns {Path}
+ */
+
+Path.prototype.fromOptions = function fromOptions(options) {
+  this.keyType = options.keyType;
+
+  this.id = options.id;
+  this.wid = options.wid;
+  this.name = options.name;
+  this.account = options.account;
+  this.branch = options.branch;
+  this.index = options.index;
+
+  this.encrypted = options.encrypted;
+  this.data = options.data;
+
+  this.type = options.type;
+  this.version = options.version;
+  this.hash = options.hash;
+
+  return this;
+};
+
+/**
+ * Instantiate path from options object.
+ * @param {Object} options
+ * @returns {Path}
+ */
+
+Path.fromOptions = function fromOptions(options) {
+  return new Path().fromOptions(options);
+};
+
 /**
  * Clone the path object.
  * @returns {Path}

lib/wallet/walletdb.js

@@ -210,7 +210,7 @@ WalletDB.prototype._init = function _init() {
 
 WalletDB.prototype._open = co(function* open() {
   yield this.db.open();
-  yield this.db.checkVersion('V', 2);
+  yield this.db.checkVersion('V', 3);
   yield this.writeGenesis();
 
   this.depth = yield this.getDepth();
@@ -1447,7 +1447,7 @@ WalletDB.prototype.writeBlock = function writeBlock(block, matches) {
   for (i = 0; i < block.hashes.length; i++) {
     hash = block.hashes[i];
     wallets = matches[i];
-    batch.put(layout.e(hash), serializeWallets(wallets));
+    batch.put(layout.e(hash), serializeInfo(wallets));
   }
 
   return batch.write();
@@ -2079,6 +2079,18 @@ function parseWallets(data) {
 }
 
 function serializeWallets(wallets) {
+  var p = new BufferWriter();
+  var i, wid;
+
+  for (i = 0; i < wallets.length; i++) {
+    wid = wallets[i];
+    p.writeU32(wid);
+  }
+
+  return p.render();
+}
+
+function serializeInfo(wallets) {
   var p = new BufferWriter();
   var i, info;
 

migrate/chaindb0to1.js

@@ -2,6 +2,8 @@ var bcoin = require('../');
 var co = bcoin.co;
 var assert = require('assert');
 var file = process.argv[2];
+var BufferReader = require('../lib/utils/reader');
+var BufferWriter = require('../lib/utils/writer');
 
 assert(typeof file === 'string', 'Please pass in a database path.');
 
@@ -51,7 +53,7 @@ var updateState = co(function* updateState() {
 
   hash = data.slice(0, 32);
 
-  p = new bcoin.writer();
+  p = new BufferWriter();
   p.writeHash(hash);
   p.writeU64(0);
   p.writeU64(0);

migrate/walletdb2to3.js

@@ -0,0 +1,262 @@
+var bcoin = require('../');
+var walletdb = require('../lib/wallet/walletdb');
+var Path = require('../lib/wallet/path');
+var Account = require('../lib/wallet/account');
+var layout = walletdb.layout;
+var co = bcoin.co;
+var assert = require('assert');
+var file = process.argv[2];
+var BufferReader = require('../lib/utils/reader');
+var BufferWriter = require('../lib/utils/writer');
+var db, batch;
+
+assert(typeof file === 'string', 'Please pass in a database path.');
+
+file = file.replace(/\.ldb$/, '');
+
+db = bcoin.ldb({
+  location: file,
+  db: 'leveldb',
+  compression: true,
+  cacheSize: 16 << 20,
+  writeBufferSize: 8 << 20,
+  createIfMissing: false,
+  bufferKeys: true
+});
+
+var updateVersion = co(function* updateVersion() {
+  var data, ver;
+
+  console.log('Checking version.');
+
+  data = yield db.get('V');
+
+  if (!data)
+    return;
+
+  ver = data.readUInt32LE(0, true);
+
+  if (ver !== 2)
+    throw Error('DB is version ' + ver + '.');
+
+  console.log('Backing up DB.');
+
+  yield db.backup(process.env.HOME + '/walletdb-bak-' + Date.now() + '.ldb');
+
+  ver = new Buffer(4);
+  ver.writeUInt32LE(3, 0, true);
+  batch.put('R', ver);
+});
+
+var updatePathMap = co(function* updatePathMap() {
+  var i, iter, item, oldPaths, oldPath, hash, path, keys, key, ring;
+  var total = 0;
+
+  iter = db.iterator({
+    gte: layout.p(constants.NULL_HASH),
+    lte: layout.p(constants.HIGH_HASH),
+    values: true
+  });
+
+  console.log('Migrating path map.');
+
+  for (;;) {
+    item = yield iter.next();
+
+    if (!item)
+      break;
+
+    total++;
+    hash = layout.pp(item.key);
+    oldPaths = parsePaths(data, hash);
+    keys = Object.keys(oldPaths);
+
+    for (i = 0; i < keys.length; i++) {
+      keys[i] = +keys[i];
+      key = keys[i];
+      oldPath = oldPaths[key];
+      path = new Path(oldPath);
+      if (path.data) {
+        if (path.encrypted) {
+          console.log(
+            'Cannot migrate encrypted import: %s (%s)',
+            path.data.toString('hex'),
+            path.toAddress().toBase58());
+          continue;
+        }
+        ring = keyFromRaw(path.data);
+        path.data = new bcoin.keyring(ring).toRaw();
+      }
+      batch.put(layout.P(key, hash), path.toRaw());
+    }
+
+    batch.put(layout.p(hash), serializeWallets(keys));
+  }
+
+  console.log('Migrated %d paths.', total);
+});
+
+var updateAccounts = co(function* updateAccounts() {
+  var total = 0;
+  var i, iter, item, account;
+
+  iter = db.iterator({
+    gte: layout.a(0, 0),
+    lte: layout.a(0xffffffff, 0xffffffff),
+    values: true
+  });
+
+  console.log('Migrating accounts.');
+
+  for (;;) {
+    item = yield iter.next();
+
+    if (!item)
+      break;
+
+    total++;
+    account = accountFromRaw(item.value, item.key);
+    account = new Account({ network: account.network, options: {} }, account);
+    batch.put(layout.a(account.wid, account.accountIndex), account.toRaw());
+  }
+
+  console.log('Migrated %d accounts.', total);
+});
+
+function pathFromRaw(data) {
+  var path = {};
+  var p = new BufferReader(data);
+
+  path.wid = p.readU32();
+  path.name = p.readVarString('utf8');
+  path.account = p.readU32();
+
+  switch (p.readU8()) {
+    case 0:
+      path.keyType = 0;
+      path.branch = p.readU32();
+      path.index = p.readU32();
+      if (p.readU8() === 1)
+        assert(false, 'Cannot migrate custom redeem script.');
+      break;
+    case 1:
+      path.keyType = 1;
+      path.encrypted = p.readU8() === 1;
+      path.data = p.readVarBytes();
+      path.branch = -1;
+      path.index = -1;
+      break;
+    default:
+      assert(false);
+      break;
+  }
+
+  path.version = p.read8();
+  path.type = p.readU8();
+
+  return path;
+}
+
+function parsePaths(data, hash) {
+  var p = new BufferReader(data);
+  var out = {};
+  var path;
+
+  while (p.left()) {
+    path = pathFromRaw(p);
+    out[path.wid] = path;
+    if (hash)
+      path.hash = hash;
+  }
+
+  return out;
+}
+
+function serializeWallets(wallets) {
+  var p = new BufferWriter();
+  var i, wid;
+
+  for (i = 0; i < wallets.length; i++) {
+    wid = wallets[i];
+    p.writeU32(wid);
+  }
+
+  return p.render();
+}
+
+function readAccountKey(key) {
+  return {
+    wid: key.readUInt32BE(1, true),
+    index: key.readUInt32E(5, true)
+  };
+}
+
+function accountFromRaw(data, dbkey) {
+  var account = {};
+  var p = new BufferReader(data);
+  var i, count, key;
+
+  dbkey = readAccountKey(dbkey);
+  account.wid = dbkey.wid;
+  account.network = bcoin.network.fromMagic(p.readU32());
+  account.name = p.readVarString('utf8');
+  account.initialized = p.readU8() === 1;
+  account.type = p.readU8();
+  account.m = p.readU8();
+  account.n = p.readU8();
+  account.witness = p.readU8() === 1;
+  account.accountIndex = p.readU32();
+  account.receiveDepth = p.readU32();
+  account.changeDepth = p.readU32();
+  account.accountKey = bcoin.hd.fromRaw(p.readBytes(82));
+  account.keys = [];
+  account.watchOnly = false;
+  account.nestedDepth = 0;
+
+  count = p.readU8();
+
+  for (i = 0; i < count; i++) {
+    key = bcoin.hd.fromRaw(p.readBytes(82));
+    account.keys.push(key);
+  }
+
+  return account;
+}
+
+function keyFromRaw(data, network) {
+  var ring = {};
+  var p = new BufferReader(data);
+  var key, script;
+
+  ring.network = bcoin.network.get(network);
+  ring.witness = p.readU8() === 1;
+
+  key = p.readVarBytes();
+
+  if (key.length === 32) {
+    ring.privateKey = key;
+    ring.publicKey = bcoin.ec.publicKeyCreate(key, true);
+  } else {
+    ring.publicKey = key;
+  }
+
+  script = p.readVarBytes();
+
+  if (script.length > 0)
+    ring.script = bcoin.script.fromRaw(script);
+
+  return ring;
+}
+
+co.spawn(function *() {
+  yield db.open();
+  batch = db.batch();
+  console.log('Opened %s.', file);
+  yield updateVersion();
+  yield updatePathMap();
+  yield updateAccounts();
+  yield batch.write();
+}).then(function() {
+  console.log('Migration complete.');
+  process.exit(0);
+});