drop references to block buffer. less mem usage. fixes.

This commit is contained in:
Christopher Jeffrey 2016-03-07 16:56:43 -08:00
parent 0104aae9c0
commit 862a412418
10 changed files with 205 additions and 97 deletions

View File

@ -46,58 +46,47 @@ Block.prototype.render = function render() {
};
Block.prototype.renderNormal = function renderNormal() {
this.getRaw();
if (!this._witnessSize)
return this._raw;
return bcoin.protocol.framer.block(this);
};
Block.prototype.renderWitness = function renderWitness() {
this.getRaw();
if (this._witnessSize)
return this._raw;
return bcoin.protocol.framer.witnessBlock(this);
};
Block.prototype.getRaw = function getRaw() {
var raw;
if (this._raw) {
assert(this._size > 0);
assert(this._witnessSize >= 0);
return this._raw;
}
if (this.hasWitness())
raw = bcoin.protocol.framer.witnessBlock(this);
else
raw = bcoin.protocol.framer.block(this);
this._raw = raw;
this._size = raw.length;
this._witnessSize = raw._witnessSize;
return this._raw;
return raw;
};
Block.prototype.getVirtualSize = function getVirtualSize() {
var size, witnessSize, base;
this.getRaw();
size = this._size;
witnessSize = this._witnessSize;
size = this.getSize();
witnessSize = this.getWitnessSize();
base = size - witnessSize;
return (base * 4 + witnessSize + 3) / 4 | 0;
};
Block.prototype.getSize = function getSize() {
return this.getRaw().length;
if (this._size == null)
this.getRaw();
return this._size;
};
Block.prototype.getWitnessSize = function getWitnessSize() {
if (this._witnessSize == null)
this.getRaw();
return this._witnessSize;
};
Block.prototype.hasWitness = function hasWitness() {
@ -272,8 +261,6 @@ Block.prototype.inspect = function inspect() {
type: this.type,
height: this.height,
hash: utils.revHex(this.hash('hex')),
reward: utils.btc(this.getReward()),
fee: utils.btc(this.getFee()),
date: new Date(this.ts * 1000).toISOString(),
version: this.version,
prevBlock: utils.revHex(this.prevBlock),

View File

@ -39,6 +39,7 @@ function BlockDB(node, options) {
this.file = bcoin.prefix + '/block-' + network.type + '.db';
this.options = options;
this.fsync = !!options.fsync;
this.node = node;
@ -134,7 +135,7 @@ BlockDB.prototype.migrate = function migrate(blockSize, compression, callback) {
BlockDB.prototype.saveBlock = function saveBlock(block, callback) {
var self = this;
var batch = this.db.batch();
var batch = this.batch();
batch.put('b/b/' + block.hash('hex'), block.toCompact());
@ -148,7 +149,7 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) {
BlockDB.prototype.removeBlock = function removeBlock(hash, callback) {
var self = this;
this._getCoinBlock(hash, function(err, block) {
this._getTXBlock(hash, function(err, block) {
var batch;
if (err)
@ -157,22 +158,15 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) {
if (!block)
return callback();
batch = self.db.batch();
batch = self.batch();
function cb(err) {
if (err)
return callback(err);
batch.del('b/b/' + block.hash('hex'));
batch.del('b/b/' + block.hash('hex'));
block.txs.forEach(function(tx, i) {
batch.del('t/t/' + tx.hash('hex'));
});
block.txs.forEach(function(tx, i) {
batch.del('t/t/' + tx.hash('hex'));
});
return callback(null, block);
}
self.disconnectBlock(hash, cb, batch);
self.disconnectBlock(block, callback, batch);
});
};
@ -189,7 +183,7 @@ BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) {
}
if (!batch)
batch = self.db.batch();
batch = self.batch();
batch.put('b/h/' + pad32(block.height), block.hash());
@ -258,9 +252,7 @@ BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) {
BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, batch) {
var self = this;
this._getCoinBlock(hash, function(err, block) {
var batch;
this._getTXBlock(hash, function(err, block) {
if (err)
return callback(err);
@ -270,7 +262,7 @@ BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, bat
}
if (!batch)
batch = self.db.batch();
batch = self.batch();
if (typeof hash === 'string')
assert(block.hash('hex') === hash);
@ -421,9 +413,8 @@ BlockDB.prototype.fillTX = function fillTX(tx, callback) {
if (err)
return next(err);
if (tx) {
if (tx)
input.output = bcoin.coin(tx, input.prevout.index);
}
next();
});
@ -699,6 +690,23 @@ BlockDB.prototype._getCoinBlock = function _getCoinBlock(hash, callback) {
});
};
BlockDB.prototype._getTXBlock = function _getTXBlock(hash, callback) {
var self = this;
if (hash instanceof bcoin.block)
return callback(null, hash);
return this.getBlock(hash, function(err, block) {
if (err)
return callback(err);
if (!block)
return callback();
return self.fillTXBlock(block, callback);
});
};
BlockDB.prototype.fillBlock = function fillBlock(block, callback) {
var self = this;
@ -731,6 +739,38 @@ BlockDB.prototype.fillBlock = function fillBlock(block, callback) {
});
};
BlockDB.prototype.fillTXBlock = function fillTXBlock(block, callback) {
var self = this;
return this.fillTXs(block.txs, function(err) {
var coins, i, tx, hash, j, input, id;
if (err)
return callback(err);
coins = {};
for (i = 0; i < block.txs.length; i++) {
tx = block.txs[i];
hash = tx.hash('hex');
for (j = 0; j < tx.inputs.length; j++) {
input = tx.inputs[j];
id = input.prevout.hash + '/' + input.prevout.index;
if (!input.output && coins[id]) {
input.output = coins[id];
delete coins[id];
}
}
for (j = 0; j < tx.outputs.length; j++)
coins[hash + '/' + j] = bcoin.coin(tx, j);
}
return callback(null, block);
});
};
BlockDB.prototype._getHash = function _getHash(height, callback) {
if (typeof height === 'string')
return callback(null, height);
@ -778,7 +818,7 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) {
block.txs[i] = tx;
next();
})
});
}, function(err) {
if (err)
return callback(err);
@ -975,6 +1015,12 @@ BlockDB.prototype.reset = function reset(height, callback, emit) {
});
};
BlockDB.prototype.batch = function batch() {
if (this.fsync)
return new utils.SyncBatch(this.db);
return this.db.batch();
};
/**
* Expose
*/

View File

@ -162,6 +162,15 @@ Chain.prototype._init = function _init() {
self.tip = tip;
self.height = tip.height;
// Start fsyncing writes once we're no
// longer dealing with historical data.
if (tip.height >= network.checkpoints.lastHeight) {
self.db.fsync = true;
if (self.blockdb)
self.blockdb.fsync = true;
}
self.loading = false;
self.emit('load');
});
@ -490,8 +499,6 @@ Chain.prototype._verify = function _verify(block, prev, callback) {
if (block.bits !== self.getTarget(prev, block)) {
utils.debug('Block is using wrong target: %s', block.rhash);
utils.debug('Has %d, expected: %d',
block.bits, self.getTarget(prev, block));
return done(null, false);
}
@ -982,6 +989,14 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) {
self.tip = entry;
self.height = entry.height;
// Start fsyncing writes once we're no
// longer dealing with historical data.
if (entry.height >= network.checkpoints.lastHeight) {
self.db.fsync = true;
if (self.blockdb)
self.blockdb.fsync = true;
}
return callback();
});
});
@ -1025,7 +1040,7 @@ Chain.prototype.reset = function reset(height, callback, force) {
self.emit('remove block', block);
});
}, function(entry) {
self.emit('remove entry', block);
self.emit('remove entry', entry);
});
};
@ -1091,25 +1106,24 @@ Chain.prototype.resetTime = function resetTime(ts, callback, force) {
if (!unlock)
return;
callback = utils.ensure(callback);
this.byTime(ts, function(err, entry) {
if (err) {
unlock();
if (callback)
callback(err);
callback(err);
return;
}
if (!entry) {
unlock();
if (callback)
callback();
callback();
return;
}
self.reset(entry.height, function(err) {
unlock();
if (callback)
callback(err);
callback(err);
}, true);
}, true);
};

View File

@ -47,6 +47,7 @@ function ChainDB(node, chain, options) {
this.fd = null;
this.loading = false;
this.loaded = false;
this.fsync = !!options.fsync;
// Keep track of block hashes in a
// bloom filter to avoid DB lookups.
@ -64,8 +65,8 @@ function ChainDB(node, chain, options) {
else
this._cacheWindow = network.block.majorityWindow + 1;
this.cacheHash = new DumbCache(this._cacheWindow * 200); // (not hashcash)
this.cacheHeight = new DumbCache(this._cacheWindow * 200);
this.cacheHash = new NullCache(this._cacheWindow * 200); // (not hashcash)
this.cacheHeight = new NullCache(this._cacheWindow * 200);
// this.cacheHash = new bcoin.lru(this._cacheWindow, function() { return 1; }); // (not hashcash)
// this.cacheHeight = new bcoin.lru(this._cacheWindow, function() { return 1; });
@ -370,7 +371,7 @@ ChainDB.prototype.save = function save(entry, callback) {
// this.bloom.add(entry.hash, 'hex');
batch = this.db.batch();
batch = this.batch();
height = new Buffer(4);
utils.writeU32(height, entry.height, 0);
@ -412,7 +413,7 @@ ChainDB.prototype.connect = function connect(block, callback, emit) {
if (!entry)
return callback();
batch = self.db.batch();
batch = self.batch();
batch.put('c/h/' + pad32(entry.height), new Buffer(entry.hash, 'hex'));
batch.put('c/t', new Buffer(entry.hash, 'hex'));
@ -438,7 +439,7 @@ ChainDB.prototype.disconnect = function disconnect(block, callback) {
if (!entry)
return callback();
batch = self.db.batch();
batch = self.batch();
batch.del('c/h/' + pad32(entry.height));
batch.put('c/t', new Buffer(entry.prevBlock, 'hex'));
@ -489,7 +490,7 @@ ChainDB.prototype.reset = function reset(block, callback, emit) {
if (!tip)
return callback();
batch = self.db.batch();
batch = self.batch();
(function next(err, tip) {
if (err)
@ -528,6 +529,12 @@ ChainDB.prototype.has = function has(height, callback) {
});
};
ChainDB.prototype.batch = function batch() {
if (this.fsync)
return new utils.SyncBatch(this.db);
return this.db.batch();
};
function DumbCache(size) {
this.data = {};
this.count = 0;
@ -573,6 +580,14 @@ DumbCache.prototype.reset = function reset() {
this.count = 0;
};
function NullCache(size) {}
NullCache.prototype.set = function set(key, value) {};
NullCache.prototype.remove = function remove(key) {};
NullCache.prototype.get = function get(key) {};
NullCache.prototype.has = function has(key) {};
NullCache.prototype.reset = function reset() {};
/**
* Expose
*/

View File

@ -38,7 +38,10 @@ CompactBlock.prototype.getCoinbaseHeight = function getCoinbaseHeight() {
};
CompactBlock.prototype.toBlock = function toBlock() {
var block = new bcoin.block(bcoin.protocol.parser.parseBlock(this._raw));
var block = bcoin.protocol.parser.parseBlock(this._raw);
delete this._raw;
assert(!block._raw);
block = new bcoin.block(block);
if (this.valid != null)
block.valid = this.valid;
return block;

View File

@ -77,7 +77,7 @@ Mempool.prototype.removeBlock = function removeBlock(block) {
self.removeTX(mtx);
});
// Add transaction back into mempool
tx = tx.clone();
// tx = tx.clone();
tx.ps = utils.now();
tx.ts = 0;
tx.block = null;

View File

@ -47,7 +47,8 @@ Fullnode.prototype._init = function _init() {
// chain, but that's only once it's being
// used for tx retrieval.
this.blockdb = new bcoin.blockdb(this, {
cache: false
cache: false,
fsync: true
});
// Mempool needs access to blockdb.
@ -57,7 +58,8 @@ Fullnode.prototype._init = function _init() {
// Chain needs access to blockdb.
this.chain = new bcoin.chain(this, {
preload: false
preload: false,
fsync: true
});
// Pool needs access to the chain.
@ -135,6 +137,7 @@ Fullnode.prototype._init = function _init() {
self.mempool.addBlock(block);
});
if (0)
this.chain.on('remove block', function(block) {
self.mempool.removeBlock(block);
self.walletdb.removeBlock(block);

View File

@ -196,7 +196,7 @@ Parser.parseVersion = function parseVersion(p) {
// User agent length
result = utils.readIntv(p, 80);
off = result.off;
agent = p.slice(off, off + result.r);
agent = p.slice(off, off + result.r).toString('ascii');
off += result.r;
// Start height
@ -229,7 +229,7 @@ Parser.parseVersion = function parseVersion(p) {
local: recv,
remote: from,
nonce: nonce,
agent: agent.toString('ascii'),
agent: agent,
height: height,
relay: relay
};
@ -249,7 +249,7 @@ Parser.parseInvList = function parseInvList(p) {
for (i = 0, off = 0; i < count; i++, off += 36) {
items.push({
type: constants.invByVal[utils.readU32(p, off)],
hash: p.slice(off + 4, off + 36)
hash: new Buffer(p.slice(off + 4, off + 36))
});
}
@ -272,7 +272,7 @@ Parser.parseMerkleBlock = function parseMerkleBlock(p) {
hashes = new Array(hashCount);
for (i = 0; i < hashCount; i++)
hashes[i] = p.slice(off + i * 32, off + (i + 1) * 32);
hashes[i] = new Buffer(p.slice(off + i * 32, off + (i + 1) * 32));
off = off + 32 * hashCount;
flagCount = utils.readIntv(p, off);
@ -282,19 +282,18 @@ Parser.parseMerkleBlock = function parseMerkleBlock(p) {
if (off + flagCount > p.length)
throw new Error('Invalid flag count');
flags = p.slice(off, off + flagCount);
flags = new Buffer(p.slice(off, off + flagCount));
return {
version: utils.read32(p, 0),
prevBlock: p.slice(4, 36),
merkleRoot: p.slice(36, 68),
prevBlock: new Buffer(p.slice(4, 36)),
merkleRoot: new Buffer(p.slice(36, 68)),
ts: utils.readU32(p, 68),
bits: utils.readU32(p, 72),
nonce: utils.readU32(p, 76),
totalTX: utils.readU32(p, 80),
hashes: hashes,
flags: flags,
_raw: p,
_size: p.length
};
};
@ -318,9 +317,9 @@ Parser.parseHeaders = function parseHeaders(p) {
start = off;
header.version = utils.read32(p, off);
off += 4;
header.prevBlock = p.slice(off, off + 32);
header.prevBlock = new Buffer(p.slice(off, off + 32));
off += 32;
header.merkleRoot = p.slice(off, off + 32);
header.merkleRoot = new Buffer(p.slice(off, off + 32));
off += 32;
header.ts = utils.readU32(p, off);
off += 4;
@ -331,7 +330,6 @@ Parser.parseHeaders = function parseHeaders(p) {
r = utils.readIntv(p, off);
header.totalTX = r.r;
off = r.off;
header._raw = p.slice(start, start + 80);
headers.push(header);
}
@ -351,7 +349,7 @@ Parser.parseBlock = function parseBlock(p) {
totalTX = result.r;
for (i = 0; i < totalTX; i++) {
tx = Parser.parseTX(p.slice(off));
tx = Parser.parseTX(p.slice(off), true);
if (!tx)
throw new Error('Invalid tx count for block');
tx._offset = off;
@ -362,14 +360,13 @@ Parser.parseBlock = function parseBlock(p) {
return {
version: utils.read32(p, 0),
prevBlock: p.slice(4, 36),
merkleRoot: p.slice(36, 68),
prevBlock: new Buffer(p.slice(4, 36)),
merkleRoot: new Buffer(p.slice(36, 68)),
ts: utils.readU32(p, 68),
bits: utils.readU32(p, 72),
nonce: utils.readU32(p, 76),
totalTX: totalTX,
txs: txs,
_raw: p,
_size: p.length,
_witnessSize: witnessSize
};
@ -412,8 +409,8 @@ Parser.parseBlockCompact = function parseBlockCompact(p) {
return {
version: version,
prevBlock: p.slice(4, 36),
merkleRoot: p.slice(36, 68),
prevBlock: new Buffer(p.slice(4, 36)),
merkleRoot: new Buffer(p.slice(36, 68)),
ts: utils.readU32(p, 68),
bits: utils.readU32(p, 72),
nonce: utils.readU32(p, 76),
@ -441,10 +438,10 @@ Parser.parseInput = function parseInput(p) {
return {
_size: off + scriptLen + 4,
prevout: {
hash: p.slice(0, 32),
hash: new Buffer(p.slice(0, 32)),
index: utils.readU32(p, 32)
},
script: bcoin.script.decode(p.slice(off, off + scriptLen)),
script: bcoin.script.decode(new Buffer(p.slice(off, off + scriptLen))),
sequence: utils.readU32(p, off + scriptLen)
};
};
@ -465,7 +462,7 @@ Parser.parseOutput = function parseOutput(p) {
return {
_size: off + scriptLen,
value: utils.read64(p, 0),
script: bcoin.script.decode(p.slice(off, off + scriptLen))
script: bcoin.script.decode(new Buffer(p.slice(off, off + scriptLen)))
};
};
@ -494,11 +491,11 @@ Parser.parseCoin = function parseCoin(p, extended) {
if (off + scriptLen > p.length - (extended ? 37 : 0))
throw new Error('Invalid utxo script length');
script = bcoin.script.decode(p.slice(off, off + scriptLen));
script = bcoin.script.decode(new Buffer(p.slice(off, off + scriptLen)));
off += scriptLen;
if (extended) {
hash = p.slice(off, off + 32);
hash = new Buffer(p.slice(off, off + 32));
off += 32;
index = utils.readU32(p, off);
@ -523,17 +520,18 @@ Parser.parseCoin = function parseCoin(p, extended) {
};
};
Parser.parseTX = function parseTX(p) {
Parser.parseTX = function parseTX(p, block) {
var off = 0;
var inCount, txIn, tx;
var outCount, txOut;
var version, locktime, i;
var raw;
if (p.length < 10)
throw new Error('Invalid tx size');
if (Parser.isWitnessTX(p))
return Parser.parseWitnessTX(p);
return Parser.parseWitnessTX(p, block);
version = utils.readU32(p, off);
off += 4;
@ -590,13 +588,18 @@ Parser.parseTX = function parseTX(p) {
locktime = utils.readU32(p, off);
off += 4;
raw = p.length !== off ? p.slice(0, off) : p;
if (block)
raw = new Buffer(raw);
return {
version: version,
inputs: txIn,
outputs: txOut,
locktime: locktime,
_witnessSize: 0,
_raw: p.length !== off ? p.slice(0, off) : p,
_raw: raw,
_size: off
};
};
@ -608,13 +611,14 @@ Parser.isWitnessTX = function isWitnessTX(p) {
return p[4] === 0 && p[5] !== 0;
};
Parser.parseWitnessTX = function parseWitnessTX(p) {
Parser.parseWitnessTX = function parseWitnessTX(p, block) {
var off = 0;
var inCount, txIn, tx;
var outCount, txOut;
var marker, flag;
var version, locktime, i;
var witnessSize = 0;
var raw;
if (p.length < 12)
throw new Error('Invalid witness tx size');
@ -699,6 +703,11 @@ Parser.parseWitnessTX = function parseWitnessTX(p) {
locktime = utils.readU32(p, off);
off += 4;
raw = p.length !== off ? p.slice(0, off) : p;
if (block)
raw = new Buffer(raw);
return {
version: version,
marker: marker,
@ -706,7 +715,7 @@ Parser.parseWitnessTX = function parseWitnessTX(p) {
inputs: txIn,
outputs: txOut,
locktime: locktime,
_raw: off !== p.length ? p.slice(0, off) : p,
_raw: raw,
_size: off,
_witnessSize: witnessSize + 2
};
@ -725,7 +734,7 @@ Parser.parseWitness = function parseWitness(p) {
chunkSize = utils.readIntv(p, off);
off = chunkSize.off;
chunkSize = chunkSize.r;
item = p.slice(off, off + chunkSize);
item = new Buffer(p.slice(off, off + chunkSize));
off += chunkSize;
witness.push(item);
if (off > p.length)
@ -768,7 +777,7 @@ Parser.parseReject = function parseReject(p) {
off += reasonLen;
data = p.slice(off, off + 32);
data = new Buffer(p.slice(off, off + 32));
return {
message: message,
@ -794,7 +803,7 @@ Parser.parseAddress = function parseAddress(p, off, full) {
services = utils.readU64(p, off);
off += 8;
ip = p.slice(off, off + 16);
ip = new Buffer(p.slice(off, off + 16));
off += 16;
port = utils.readU16BE(p, off);

View File

@ -720,6 +720,8 @@ TXPool.prototype.unconfirm = function unconfirm(hash, callback) {
if (hash.hash)
hash = hash.hash('hex');
callback = utils.ensure(callback);
this.getTX(hash, function(err, tx) {
if (err)
return callback(err);

View File

@ -14,6 +14,10 @@ var util = require('util');
* Utils
*/
utils.slice = function slice(buf, start, end) {
return new Buffer(buf.slice(start, end));
};
utils.toBuffer = function toBuffer(msg, enc) {
if (Buffer.isBuffer(msg))
return msg;
@ -1730,3 +1734,28 @@ utils.pad32 = function pad32(num) {
assert(num.length === 10);
return num;
};
function SyncBatch(db) {
this.db = db;
this.ops = [];
}
SyncBatch.prototype.put = function put(key, value) {
this.ops.push({ type: 'put', key: key, value: value, sync: true });
};
SyncBatch.prototype.del = function del(key) {
this.ops.push({ type: 'del', key: key, sync: true });
};
SyncBatch.prototype.write = function write(callback) {
this.db.batch(this.ops, { sync: true }, callback);
this.ops.length = 0;
delete this.ops;
};
SyncBatch.prototype.clear = function clear() {
this.ops.length = 0;
};
utils.SyncBatch = SyncBatch;