remove datastore and tx-pool.js

This commit is contained in:
Christopher Jeffrey 2016-03-29 19:31:26 -07:00
parent 6655e57053
commit 498944781e
2 changed files with 0 additions and 1132 deletions

View File

@ -1,657 +0,0 @@
/**
* datastore.js - storage
* Copyright (c) 2014-2015, Fedor Indutny (MIT License)
* https://github.com/indutny/bcoin
*/
var EventEmitter = require('events').EventEmitter;
var fs = require('fs');
var bcoin = require('../bcoin');
var network = bcoin.protocol.network;
var utils = require('./utils');
var assert = utils.assert;
var pad32 = utils.pad32;
var MAX_FILE_SIZE = 128 * 1024 * 1024;
// var MAX_FILE_SIZE = 10 * 1024 * 1024;
var NULL_CHUNK = new Buffer([0xff, 0xff, 0xff, 0xff]);
/**
* DataStore
*/
function DataStore(db, options) {
var self = this;
if (!(this instanceof DataStore))
return new DataStore(db, options);
EventEmitter.call(this);
if (!options)
options = {};
this.options = options;
this.dir = options.name;
bcoin.ensurePrefix();
if (!this.dir)
this.dir = bcoin.prefix + '/store-' + network.type + '.db';
this._db = db;
this.db = this;
this.busy = false;
this.jobs = [];
this.fileIndex = -1;
// Keep a pool of FDs open for disk cache benefits
this.pool = new bcoin.lru(50, 1, function(key, value) {
fs.close(value.fd, function(err) {
if (err)
self.emit('error', err);
});
});
this._init();
}
utils.inherits(DataStore, EventEmitter);
DataStore.prototype._lock = function _lock(func, args, force) {
var self = this;
var called;
if (force) {
assert(this.busy);
return function unlock() {
assert(!called);
called = true;
};
}
if (this.busy) {
this.jobs.push([func, args]);
return;
}
this.busy = true;
return function unlock() {
var item;
assert(!called);
called = true;
self.busy = false;
if (self.jobs.length === 0) {
self.emit('flush');
return;
}
item = self.jobs.shift();
item[0].apply(self, item[1]);
};
};
DataStore.prototype._exists = function _exists(callback) {
fs.stat(this.dir, function(err) {
if (err && err.code !== 'ENOENT')
return callback(err);
return callback(null, err == null);
});
};
DataStore.prototype._ensure = function _ensure(callback) {
var self = this;
this._exists(function(err, result) {
if (err)
return callback(err);
if (result)
return callback();
return fs.mkdir(self.dir, 0750, callback);
});
};
DataStore.prototype._init = function _init(callback) {
var self = this;
callback = utils.ensure(callback);
return this._ensure(function(err) {
if (err)
return callback(err);
self.getLastIndex(function(err, index) {
if (err)
return callback(err);
self.fileIndex = index;
return callback();
});
});
};
DataStore.prototype.allocatePage = function allocatePage(callback) {
var self = this;
var index = this.fileIndex + 1;
fs.writeFile(this.dir + '/f' + pad32(index), new Buffer([]), function(err) {
if (err)
return callback(err);
self.openFile(index, function(err, fd, size, index) {
if (err)
return callback(err);
self.fileIndex = index;
return callback(null, fd, size, index);
});
});
};
DataStore.prototype.openFile = function openFile(index, callback) {
var self = this;
var entry = this.pool.get(index);
if (entry)
return callback(null, entry.fd, entry.size, index);
fs.open(this.dir + '/f' + pad32(index), 'r+', function(err, fd) {
if (err)
return callback(err);
fs.fstat(fd, function(err, stat) {
if (err)
return callback(err);
self.pool.set(index, { fd: fd, size: stat.size });
return callback(null, fd, stat.size, index);
});
});
};
DataStore.prototype.currentFile = function currentFile(callback) {
if (this.fileIndex === -1)
return this.allocatePage(callback);
return this.openFile(this.fileIndex, callback);
};
DataStore.prototype.getLastIndex = function getLastIndex(callback) {
var i, max, index;
fs.readdir(this.dir, function(err, list) {
if (err)
return callback(err);
max = -1;
for (i = 0; i < list.length; i++) {
if (!/^f\d{10}$/.test(list[i]))
continue;
index = +list[i].substring(1);
if (index > max)
max = index;
}
return callback(null, max);
});
};
DataStore.prototype.close = function close(callback) {
return callback();
};
DataStore.prototype.getData = function get(off, callback) {
var self = this;
callback = utils.ensure(callback);
off = this.parseOffset(off);
this.openFile(off.fileIndex, function(err, fd, fsize) {
if (err)
return callback(err);
return self.read(fd, off.offset, off.size, function(err, data) {
if (err)
return callback(err);
return callback(null, data);
});
});
};
DataStore.prototype.get = function get(key, callback) {
var self = this;
return this._db.get(key, function(err, offset) {
if (err)
return callback(err);
if (isDirect(key))
return callback(null, offset);
return self.getData(offset, callback);
});
};
DataStore.prototype.batch = function batch(ops, options, callback) {
var batch;
if (!callback) {
callback = options;
options = null;
}
if (!options)
options = {};
batch = new Batch(this, options);
if (ops) {
batch.ops = ops;
return batch.write(callback);
}
return batch;
};
function Batch(store, options) {
this.options = options;
this.ops = [];
this.store = store;
this._db = store._db;
}
Batch.prototype.put = function(key, value) {
this.ops.push({ type: 'put', key: key, value: value });
};
Batch.prototype.del = function del(key) {
this.ops.push({ type: 'del', key: key });
};
Batch.prototype.write = function write(callback) {
var self = this;
var batch;
if (!this._db)
return callback(new Error('Already written.'));
batch = this.options.sync
? utils.syncBatch(this._db)
: this._db.batch();
if (this.options.sync)
this._db.fsync = true;
utils.forEachSerial(this.ops, function(op, next) {
if (op.type === 'put') {
if (isDirect(op.key)) {
batch.put(op.key, op.value);
return next();
}
return self.store.putData(op.value, function(err, offset) {
if (err)
return callback(err);
batch.put(op.key, offset);
next();
});
}
if (op.type === 'del') {
if (isDirect(op.key)) {
batch.del(op.key);
return next();
}
return self._db.get(op.key, function(err, offset) {
if (err && err.type !== 'NotFoundError')
return callback(err);
if (!offset)
return next();
batch.del(op.key);
self.store.delData(offset, next);
});
}
return callback(new Error('Bad op type.'));
}, function(err) {
self.ops.length = 0;
delete self.ops;
delete self._db;
delete self.store;
if (err)
return callback(err);
return batch.write(callback);
});
};
DataStore.prototype.iterator = function iterator(options) {
return new Iterator(this, options);
};
function Iterator(store, options) {
this.store = store;
this._db = store._db;
if (!options)
options = {};
if (options.keys === false)
options.keys = true;
this.options = options;
this.iterator = this._db.iterator(options);
}
Iterator.prototype.seek = function seek(key) {
return this.iterator.seek(key);
};
// Store coins, chain entries, dummies, lookup
// hashes directly in the db (unless they're
// the same length as offset).
function isDirect(key) {
return !/^(b\/b\/|t\/t\/|c\/c\/|u\/t\/)/.test(key);
}
Iterator.prototype.next = function next(callback) {
var self = this;
return this.iterator.next(function(err, key, value) {
if (err)
return callback(err);
if (self.options.values !== false && value) {
if (isDirect(key))
return callback(null, key, value);
return self.store.getData(value, function(err, data) {
if (err)
return callback(err);
return callback(null, key, data);
});
}
return callback(null, key, value);
});
};
Iterator.prototype.end = function end(callback) {
var ret = this.iterator.end(callback);
delete this.iterator;
delete this.store;
delete this._db;
return ret;
};
DataStore.prototype.putData = function putData(data, callback, force) {
var self = this;
var offset;
var unlock = this._lock(putData, [data, callback], force);
if (!unlock)
return;
callback = utils.wrap(callback, unlock);
this.currentFile(function(err, fd, fsize, index) {
if (err)
return callback(err);
return self.write(fd, fsize, data, function(err, written) {
if (err)
return callback(err);
offset = self.createOffset(index, fsize, written);
fsize += written;
// tx1 -> tx1-start/undo -> tx2 -> tx2-start/undo
return self.write(fd, fsize, offset, function(err, written) {
if (err)
return callback(err);
fsize += written;
if (self.pool.has(index))
self.pool.get(index).size = fsize;
if (fsize > MAX_FILE_SIZE) {
return self.allocatePage(function(err) {
if (err)
return callback(err);
return callback(null, offset);
});
}
return callback(null, offset);
});
});
});
};
DataStore.prototype.put = function put(key, value, callback) {
var self = this;
var unlock = this._lock(put, [key, value, callback]);
if (!unlock)
return;
callback = utils.wrap(callback, unlock);
if (isDirect(key))
return this._db.put(key, value, callback);
return this.putData(value, function(err, offset) {
if (err)
return callback(err);
return self._db.put(key, offset, callback);
}, true);
};
DataStore.prototype.readUndo = function readUndo(index, offset, callback) {
var self = this;
return this.openFile(index, function(err, fd, fsize) {
if (err)
return callback(err);
return self.read(fd, offset - 12, 12, function(err, data) {
if (err)
return callback(err);
return callback(null, self.parseOffset(data));
});
});
};
DataStore.prototype.delData = function delData(off, callback, force) {
var self = this;
var index, offset, size;
var unlock = this._lock(delData, [off, callback], force);
if (!unlock)
return;
callback = utils.wrap(callback, unlock);
off = this.parseOffset(off);
index = off.fileIndex;
offset = off.offset;
size = off.size;
return this.openFile(index, function(err, fd, fsize) {
if (err)
return callback(err);
// Overwrite the "fileIndex" in the undo chunk
return self.write(fd, offset + size, NULL_CHUNK, function(err) {
if (err)
return callback(err);
if (offset + size + 12 !== fsize)
return callback();
// If we're deleting the last record, traverse
// through the reverse linked list of undo offsets
// until we hit a record that isn't deleted.
// Truncate to the last deleted record's offset.
(function next() {
if (offset === 0)
return done();
self.readUndo(index, offset, function(err, undo) {
if (err)
return callback(err);
if (undo.fileIndex !== 0xffffffff)
return done();
offset = undo.offset;
if (offset === 0)
return done();
return next();
});
})();
function done() {
// Delete the file if nothing is in it.
if (offset === 0) {
self.pool.remove(index);
return fs.unlink(self.dir + '/f' + pad32(index), callback);
}
self.truncate(index, offset + 12, callback);
}
});
});
};
DataStore.prototype.del = function del(key, callback, force) {
var self = this;
var unlock = this._lock(del, [key, callback], force);
if (!unlock)
return;
callback = utils.wrap(callback, unlock);
if (isDirect(key))
return this._db.del(key, callback);
this._db.get(key, function(err, off) {
if (err && err.type !== 'NotFoundError')
return callback(err);
if (!off)
return callback();
self.delData(off, function(err) {
if (err)
return callback(err);
self._db.del(key, callback);
}, true);
});
};
DataStore.prototype.createOffset = function createOffset(fileIndex, offset, size) {
var buf = new Buffer(12);
utils.writeU32(buf, fileIndex, 0);
utils.writeU32(buf, offset, 4);
utils.writeU32(buf, size, 8);
return buf;
};
DataStore.prototype.parseOffset = function parseOffset(data) {
return {
fileIndex: utils.readU32(data, 0),
offset: utils.readU32(data, 4),
size: utils.readU32(data, 8)
};
};
DataStore.prototype.truncate = function truncate(index, size, callback) {
var self = this;
callback = utils.ensure(callback);
this.openFile(index, function(err, fd, fsize) {
if (err)
return callback(err);
fs.ftruncate(fd, size, function(err) {
if (err)
return callback(err);
if (self.pool.has(index))
self.pool.get(index).size = size;
return callback();
});
});
};
DataStore.prototype._ioError = function _ioError(name, size, offset) {
return new Error(name
+ '() failed at offset '
+ offset
+ ' with '
+ size
+ ' bytes left.');
};
DataStore.prototype.read = function read(fd, offset, size, callback) {
var self = this;
var index = 0;
var data;
callback = utils.ensure(callback);
assert(!(offset < 0 || offset == null));
data = new Buffer(size);
(function next() {
fs.read(fd, data, index, size, offset, function(err, bytes) {
if (err)
return callback(err);
if (!bytes)
return callback(self._ioError('read', size, offset));
index += bytes;
size -= bytes;
offset += bytes;
if (index === data.length)
return callback(null, data);
next();
});
})();
};
DataStore.prototype.write = function write(fd, offset, data, callback) {
var self = this;
var size = data.length;
var index = 0;
callback = utils.ensure(callback);
assert(!(offset < 0 || offset == null));
(function next() {
fs.write(fd, data, index, size, offset, function(err, bytes) {
if (err)
return callback(err);
if (!bytes)
return callback(self._ioError('write', size, offset));
index += bytes;
size -= bytes;
offset += bytes;
if (index === data.length) {
if (!self.fsync)
return callback(null, index);
return fs.fsync(fd, function(err) {
if (err)
return callback(err);
return callback(null, index);
});
}
next();
});
})();
};
module.exports = DataStore;

View File

@ -1,475 +0,0 @@
/**
* tx-pool.js - transaction pool for bcoin
* Copyright (c) 2014-2015, Fedor Indutny (MIT License)
* https://github.com/indutny/bcoin
*/
var bn = require('bn.js');
var bcoin = require('../bcoin');
var utils = require('./utils');
var assert = bcoin.utils.assert;
var EventEmitter = require('events').EventEmitter;
/**
* TXPool
*/
function TXPool(wallet, txs) {
if (!(this instanceof TXPool))
return new TXPool(wallet, txs);
EventEmitter.call(this);
this._wallet = wallet;
this._all = {};
this._coins = {};
this._orphans = {};
this._lastTs = 0;
this._lastHeight = 0;
this._loaded = false;
this._addresses = {};
this._sent = new bn(0);
this._received = new bn(0);
this._balance = new bn(0);
this._init(txs);
}
utils.inherits(TXPool, EventEmitter);
TXPool.prototype._init = function _init(txs) {
var self = this;
if (!txs)
return;
utils.nextTick(function() {
self.populate(txs);
});
};
TXPool.prototype.populate = function populate(txs) {
txs.forEach(function(tx) {
this.add(tx, true);
}, this);
};
TXPool.prototype.add = function add(tx, noWrite) {
var hash = tx.hash('hex');
var updated = false;
var i, j, input, output, coin, orphan;
var key, orphans, some;
this._wallet.fillCoins(tx);
if (!this._wallet.ownInput(tx) && !this._wallet.ownOutput(tx))
return false;
if (tx.type === 'mtx')
tx = tx.toTX();
// Ignore stale pending transactions
if (tx.ts === 0 && tx.ps + 2 * 24 * 3600 < utils.now()) {
this._removeTX(tx, noWrite);
return;
}
// Do not add TX two times
if (this._all[hash]) {
// Transaction was confirmed, update it in storage
if (tx.ts !== 0 && this._all[hash].ts === 0) {
this._all[hash].ts = tx.ts;
this._all[hash].block = tx.block;
this._all[hash].height = tx.height;
this._all[hash].index = tx.index;
this._all[hash].outputs.forEach(function(output, i) {
var key = hash + '/' + i;
if (this._coins[key])
this._coins[key].height = tx.height;
}, this);
this._storeTX(hash, tx, noWrite);
this._lastTs = Math.max(tx.ts, this._lastTs);
this._lastHeight = Math.max(tx.height, this._lastHeight);
this.emit('update', this._lastTs, this._lastHeight, tx);
this.emit('confirmed', tx);
}
return false;
}
this._all[hash] = tx;
// Consume unspent money or add orphans
for (i = 0; i < tx.inputs.length; i++) {
input = tx.inputs[i];
key = input.prevout.hash + '/' + input.prevout.index;
coin = this._coins[key];
if (coin) {
// Add TX to inputs and spend money
input.coin = coin;
assert(input.prevout.hash === coin.hash);
assert(input.prevout.index === coin.index);
// Skip invalid transactions
if (!tx.verify(i))
return;
this._addInput(tx, i);
delete this._coins[key];
updated = true;
continue;
}
// Only add orphans if this input is ours.
if (!this._wallet.ownInput(input))
continue;
// Add orphan, if no parent transaction is yet known
orphan = { tx: tx, index: i };
if (this._orphans[key])
this._orphans[key].push(orphan);
else
this._orphans[key] = [orphan];
}
// Add unspent outputs or resolve orphans
for (i = 0; i < tx.outputs.length; i++) {
output = tx.outputs[i];
// Do not add unspents for outputs that aren't ours.
if (!this._wallet.ownOutput(tx, i))
continue;
coin = bcoin.coin(tx, i);
this._addOutput(tx, i);
key = hash + '/' + i;
orphans = this._orphans[key];
// Add input to orphan
if (orphans) {
some = false;
for (j = 0; j < orphans.length; j++) {
orphan = orphans[j];
orphan.tx.inputs[orphan.index].coin = coin;
assert(orphan.tx.inputs[orphan.index].prevout.hash === hash);
assert(orphan.tx.inputs[orphan.index].prevout.index === i);
// Verify that input script is correct, if not - add
// output to unspent and remove orphan from storage
if (orphan.tx.verify(orphan.index)) {
this._addInput(orphan.tx, orphan.index);
some = true;
break;
}
this._removeTX(orphan.tx, noWrite);
}
if (!some)
orphans = null;
}
delete this._orphans[key];
if (!orphans) {
this._coins[key] = coin;
updated = true;
}
}
this._lastTs = Math.max(tx.ts, this._lastTs);
this._lastHeight = Math.max(tx.height, this._lastHeight);
if (updated)
this.emit('update', this._lastTs, this._lastHeight, tx);
this.emit('tx', tx);
if (tx.ts !== 0)
this.emit('confirmed', tx);
this._storeTX(hash, tx, noWrite);
return true;
};
TXPool.prototype.getTX = function getTX(hash) {
return this._all[hash];
};
TXPool.prototype.getCoin = function getCoin(hash, index) {
return this._coins[hash + '/' + index];
};
TXPool.prototype._storeTX = function _storeTX(hash, tx, noWrite) {
var self = this;
if (noWrite)
return;
this._wallet.save(function(err) {
if (err)
self.emit('error', err);
});
};
TXPool.prototype._removeTX = function _removeTX(tx, noWrite) {
var self = this;
var hash = tx.hash('hex');
var key, i;
for (i = 0; i < tx.outputs.length; i++) {
key = hash + '/' + i;
if (this._coins[key]) {
delete this._coins[key];
this._removeOutput(tx, i);
}
}
// delete this._all[hash];
if (noWrite)
return;
this._wallet.save(function(err) {
if (err)
self.emit('error', err);
});
};
TXPool.prototype.removeTX = function removeTX(hash) {
var tx, input, updated, i, key;
if (hash.hash)
hash = hash('hex');
tx = this._all[hash];
if (!tx)
return false;
this._removeTX(tx, false);
delete this._all[hash];
for (i = 0; i < tx.inputs.length; i++) {
input = tx.inputs[i];
if (!input.coin || !this._wallet.ownOutput(input.coin))
continue;
this._removeInput(input);
key = input.prevout.hash + '/' + input.prevout.index;
this._coins[key] = input.coin;
updated = true;
}
if (updated)
this.emit('update', this._lastTs, this._lastHeight);
};
TXPool.prototype.unconfirm = function unconfirm(hash) {
var tx;
if (hash.hash)
hash = hash('hex');
tx = this._all[hash];
if (!tx)
return false;
if (this._lastHeight >= tx.height)
this._lastHeight = tx.height;
if (this._lastTs >= tx.ts)
this._lastTs = tx.ts;
tx.ps = utils.now();
tx.ts = 0;
tx.block = null;
tx.height = -1;
tx.index = -1;
tx.outputs.forEach(function(output, i) {
var key = hash + '/' + i;
if (this._coins[key])
this._coins[key].height = -1;
}, this);
this._storeTX(hash, tx);
this._lastTs = Math.max(tx.ts, this._lastTs);
this._lastHeight = Math.max(tx.height, this._lastHeight);
this.emit('update', this._lastTs, this._lastHeight, tx);
this.emit('unconfirmed', tx);
};
TXPool.prototype._addOutput = function _addOutput(tx, i, remove) {
var output, address;
if ((tx instanceof bcoin.output) || (tx instanceof bcoin.coin))
output = tx;
else
output = tx.outputs[i];
if (!this._wallet.ownOutput(output))
return;
address = output.getAddress();
if (!this._addresses[address]) {
this._addresses[address] = {
received: new bn(0),
sent: new bn(0),
balance: new bn(0)
};
}
if (!remove) {
this._addresses[address].balance.iadd(output.value);
this._addresses[address].received.iadd(output.value);
} else {
this._addresses[address].balance.isub(output.value);
this._addresses[address].received.isub(output.value);
}
if (!remove) {
this._balance.iadd(output.value);
this._received.iadd(output.value);
} else {
this._balance.isub(output.value);
this._received.isub(output.value);
}
};
TXPool.prototype._removeOutput = function _removeOutput(tx, i) {
return this._addOutput(tx, i, true);
};
TXPool.prototype._addInput = function _addInput(tx, i, remove) {
var input, prev, address;
if (tx instanceof bcoin.input)
input = tx;
else
input = tx.inputs[i];
assert(input.coin);
if (!this._wallet.ownOutput(input.coin))
return;
prev = input.coin;
address = prev.getAddress();
if (!this._addresses[address]) {
this._addresses[address] = {
received: new bn(0),
sent: new bn(0),
balance: new bn(0)
};
}
if (!remove) {
this._addresses[address].balance.isub(prev.value);
this._addresses[address].sent.iadd(prev.value);
} else {
this._addresses[address].balance.iadd(prev.value);
this._addresses[address].sent.isub(prev.value);
}
if (!remove) {
this._balance.isub(prev.value);
this._sent.iadd(prev.value);
} else {
this._balance.iadd(prev.value);
this._sent.isub(prev.value);
}
};
TXPool.prototype._removeInput = function _removeInput(tx, i) {
return this._addInput(tx, i, true);
};
TXPool.prototype.getAll = function getAll(address) {
return Object.keys(this._all).map(function(key) {
return this._all[key];
}, this).filter(function(tx) {
if (address) {
if (!tx.testInputs(address) && !tx.testOutputs(address))
return false;
}
return true;
});
};
TXPool.prototype.getCoins = function getCoins(address) {
return Object.keys(this._coins).map(function(key) {
return this._coins[key];
}, this).filter(function(coin) {
if (address) {
if (!coin.test(address))
return false;
}
return true;
});
};
TXPool.prototype.getPending = function getPending(address) {
return Object.keys(this._all).map(function(key) {
return this._all[key];
}, this).filter(function(tx) {
if (address) {
if (!tx.testInputs(address) && !tx.testOutputs(address))
return false;
}
return tx.ts === 0;
});
};
TXPool.prototype.getSent = function getSent(address) {
if (address) {
if (this._addresses[address])
return this._addresses[address].sent.clone();
return new bn(0);
}
return this._sent.clone();
};
TXPool.prototype.getReceived = function getReceived(address) {
if (address) {
if (this._addresses[address])
return this._addresses[address].received.clone();
return new bn(0);
}
return this._received.clone();
};
TXPool.prototype.getBalance = function getBalance(address) {
if (address) {
if (this._addresses[address])
return this._addresses[address].balance.clone();
return new bn(0);
}
return this._balance.clone();
};
TXPool.prototype.getBalanceCoins = function getBalanceCoins(address) {
var acc = new bn(0);
var coin = this.getCoins(address);
if (coin.length === 0)
return acc;
return coin.reduce(function(acc, coin) {
return acc.iadd(coin.value);
}, acc);
};
/**
* Expose
*/
module.exports = TXPool;