RanchiMall/flocore-node
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #104 from matiu/feature/fix-sync

Browse Source 
Feature/fix sync
This commit is contained in:
Gustavo Maximiliano Cortez 2014-05-30 00:11:36 -03:00
commit 8a0ab018b6
8 changed files with 213 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/addresses.js
View File

@ -53,7 +53,7 @@ exports.show = function(req, res, next) {
} else {
return res.jsonp(a.getObj());
}
}, {noTxList: req.query.noTxList});
}, {txLimit: req.query.noTxList?0:-1});
}
};

46
app/models/Address.js
View File

@ -104,13 +104,13 @@ Address.prototype._addTxItem = function(txItem, txList) {
add=1;
if (txList)
txList.push({txid: txItem.txid, ts: txItem.ts});
txList.push(txItem.txid);
}
// Spent tx
if (txItem.spentTxId && !seen[txItem.spentTxId] ) {
if (txList) {
txList.push({txid: txItem.spentTxId, ts: txItem.spentTs});
txList.push(txItem.spentTxId);
}
seen[txItem.spentTxId]=1;
addSpend=1;
@ -140,29 +140,16 @@ Address.prototype._addTxItem = function(txItem, txList) {
}
};
Address.prototype._setTxs = function(txs) {
// sort input and outputs togheter
txs.sort(
function compare(a,b) {
if (a.ts < b.ts) return 1;
if (a.ts > b.ts) return -1;
return 0;
});
this.transactions = txs.map(function(i) { return i.txid; } );
};
// opts are
// .noTxList
// .onlyUnspent
// .noSortTxs
// .txLimit (=0 -> no txs, => -1 no limit)
//
Address.prototype.update = function(next, opts) {
var self = this;
if (!self.addrStr) return next();
opts = opts || {};
var txList = opts.noTxList ? null : [];
var txList = opts.txLimit === 0 ? null: [];
var tDb = TransactionDb;
var bDb = BlockDb;
tDb.fromAddr(self.addrStr, function(err,txOut){
@ -172,8 +159,8 @@ Address.prototype.update = function(next, opts) {
if (err) return next(err);
tDb.cacheConfirmations(txOut, function(err) {
// console.log('[Address.js.161:txOut:]',txOut); //TODO
if (err) return next(err);
if (opts.onlyUnspent) {
txOut = txOut.filter(function(x){
return !x.spentTxId;
@ -197,32 +184,13 @@ Address.prototype.update = function(next, opts) {
txOut.forEach(function(txItem){
self._addTxItem(txItem, txList);
});
if (txList && !opts.noSortTxs)
self._setTxs(txList);
if (txList) self.transactions = txList;
return next();
}
});
});
});
};
Address.prototype.getUtxo = function(next) {
var self = this;
var tDb = TransactionDb;
var bDb = BlockDb;
var ret;
if (!self.addrStr) return next(new Error('no error'));
tDb.fromAddr(self.addrStr, function(err,txOut){
if (err) return next(err);
var unspent = txOut.filter(function(x){
return !x.spentTxId;
});
bDb.fillConfirmations(unspent, function() {
});
});
};
module.exports = require('soop')(Address);

15
lib/BlockDb.js
View File

@ -283,7 +283,7 @@ BlockDb.prototype.fromHashWithInfo = function(hash, cb) {
self.getHeight(hash, function(err, height) {
if (err) return cb(err);
info.isMainChain = height ? true : false;
info.isMainChain = height>=0 ? true : false;
return cb(null, {
hash: hash,
@ -326,6 +326,7 @@ BlockDb.prototype._fillConfirmationsOneSpent = function(o, chainHeight, cb) {
if (o.multipleSpentAttempts) {
async.eachLimit(o.multipleSpentAttempts, CONCURRENCY,
function(oi, e_c) {
// Only one will be confirmed
self.getBlockForTx(oi.spentTxId, function(err, hash, height) {
if (err) return;
if (height>=0) {
@ -417,7 +418,17 @@ BlockDb.prototype.migrateV02cleanup = function(cb) {
end: k + '~'
})
.pipe(d.createWriteStream({type:'del'}))
.on('close',cb);
.on('close', function(err){
if (err) return cb(err);
var k = 'txa-';
var d = self.txDb._db;
d.createReadStream({
start: k,
end: k + '~'
})
.pipe(d.createWriteStream({type:'del'}))
.on('close', cb);
});
});
};

165
lib/BlockExtractor.js
View File

@ -9,62 +9,53 @@ var bitcore = require('bitcore'),
async = require('async');
function BlockExtractor(dataDir, network) {
var self = this;
var path = dataDir + '/blocks/blk*.dat';
self.dataDir = dataDir;
self.files = glob.sync(path);
self.nfiles = self.files.length;
this.dataDir = dataDir;
this.files = glob.sync(path);
this.nfiles = this.files.length;
if (self.nfiles === 0)
if (this.nfiles === 0)
throw new Error('Could not find block files at: ' + path);
self.currentFileIndex = 0;
self.isCurrentRead = false;
self.currentBuffer = null;
self.currentParser = null;
self.network = network === 'testnet' ? networks.testnet: networks.livenet;
self.magic = self.network.magic.toString('hex');
this.currentFileIndex = 0;
this.isCurrentRead = false;
this.currentBuffer = null;
this.currentParser = null;
this.network = network === 'testnet' ? networks.testnet: networks.livenet;
this.magic = this.network.magic.toString('hex');
}
BlockExtractor.prototype.currentFile = function() {
var self = this;
return self.files[self.currentFileIndex];
return this.files[this.currentFileIndex];
};
BlockExtractor.prototype.nextFile = function() {
var self = this;
if (self.currentFileIndex < 0) return false;
if (this.currentFileIndex < 0) return false;
var ret = true;
self.isCurrentRead = false;
self.currentBuffer = null;
self.currentParser = null;
this.isCurrentRead = false;
this.currentBuffer = null;
this.currentParser = null;
if (self.currentFileIndex < self.nfiles - 1) {
self.currentFileIndex++;
if (this.currentFileIndex < this.nfiles - 1) {
this.currentFileIndex++;
}
else {
self.currentFileIndex=-1;
this.currentFileIndex=-1;
ret = false;
}
return ret;
};
BlockExtractor.prototype.readCurrentFileSync = function() {
var self = this;
if (this.currentFileIndex < 0 || this.isCurrentRead) return;
if (self.currentFileIndex < 0 || self.isCurrentRead) return;
this.isCurrentRead = true;
self.isCurrentRead = true;
var fname = self.currentFile();
var fname = this.currentFile();
if (!fname) return;
@ -81,80 +72,70 @@ BlockExtractor.prototype.readCurrentFileSync = function() {
fs.readSync(fd, buffer, 0, size, 0);
self.currentBuffer = buffer;
self.currentParser = new Parser(buffer);
this.currentBuffer = buffer;
this.currentParser = new Parser(buffer);
};
BlockExtractor.prototype.getNextBlock = function(cb) {
var self = this;
BlockExtractor.prototype._getMagic = function() {
if (!this.currentParser)
return null;
var byte0 = this.currentParser ? this.currentParser.buffer(1).toString('hex') : null;
// Grab 3 bytes from block without removing them
var p = this.currentParser.pos;
var bytes123 = this.currentParser.subject.toString('hex',p,p+3);
var magic = byte0 + bytes123;
if (magic !=='00000000' && magic !== this.magic) {
if(this.errorCount++ > 4)
throw new Error('CRITICAL ERROR: Magic number mismatch: ' +
magic + '!=' + this.magic);
magic=null;
}
if (magic==='00000000')
magic =null;
return magic;
};
BlockExtractor.prototype.getNextBlock = function(cb) {
var b;
var magic;
async.series([
function (a_cb) {
var isFinished = 0;
async.whilst(
function() {
return (!magic || magic === '00000000');
},
function(w_cb) {
magic = null;
while(!magic && !isFinished) {
this.readCurrentFileSync();
magic= this._getMagic();
self.readCurrentFileSync();
if (self.currentFileIndex < 0) return cb();
if (!this.currentParser || this.currentParser.eof() ) {
var byte0 = self.currentParser ? self.currentParser.buffer(1).toString('hex') : null;
if (this.nextFile()) {
console.log('Moving forward to file:' + this.currentFile() );
magic = null;
} else {
console.log('Finished all files');
isFinished = 1;
}
}
}
if (isFinished)
return cb();
if (byte0) {
// Grab 3 bytes from block without removing them
var p = self.currentParser.pos;
var bytes123 = self.currentParser.subject.toString('hex',p,p+3);
magic = byte0 + bytes123;
// Remove 3 bytes from magic and spacer
this.currentParser.buffer(3+4);
if (magic !=='00000000' && magic !== self.magic) {
if (self.errorCount++ > 4)
return cb(new Error('CRITICAL ERROR: Magic number mismatch: ' +
magic + '!=' + self.magic));
magic=null;
}
}
if (!self.currentParser || self.currentParser.eof() ) {
if (self.nextFile())
console.log('Moving forward to file:' + self.currentFile() );
else
console.log('Finished all files');
magic = null;
return w_cb();
}
else {
return w_cb();
}
}, a_cb);
},
function (a_cb) {
if (!magic) return a_cb();
// Remove 3 bytes from magic and spacer
self.currentParser.buffer(3+4);
return a_cb();
},
function (a_cb) {
if (!magic) return a_cb();
b = new Block();
b.parse(self.currentParser);
b.getHash();
self.errorCount=0;
return a_cb();
},
], function(err) {
return cb(err,b);
});
b = new Block();
b.parse(this.currentParser);
b.getHash();
this.errorCount=0;
return cb(null,b);
};
module.exports = require('soop')(BlockExtractor);

10
lib/HistoricSync.js
View File

@ -15,6 +15,10 @@ var bitcoreUtil = bitcore.util;
var logger = require('./logger').logger;
var info = logger.info;
var error = logger.error;
var PERCENTAGE_TO_START_FROM_RPC = 1.1;
// TODO TODO TODO
//var PERCENTAGE_TO_START_FROM_RPC = 0.98;
// var Deserialize = require('bitcore/Deserialize');
var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@ -244,7 +248,7 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) {
var self = this;
if ( opts.forceRPC || !config.bitcoind.dataDir ||
self.height > self.blockChainHeight * 0.9) return next();
self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next();
try {
@ -279,6 +283,9 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) {
});
});
}, function(err){
console.log('\tFOUND Starting Block!');
// TODO SET HEIGHT
return next(err);
});
});
@ -394,6 +401,7 @@ HistoricSync.prototype.start = function(opts, next) {
function (w_cb) {
self.getFn(function(err,blockInfo) {
if (err) return w_cb(self.setError(err));
if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) {
self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) {
if (err) return w_cb(self.setError(err));

158
lib/TransactionDb.js
View File

@ -3,13 +3,17 @@
var imports = require('soop').imports();
// to show tx outs
var OUTS_PREFIX = 'txo-'; //txo-<txid>-<n> => [addr, btc_sat]
var SPENT_PREFIX = 'txs-'; //txs-<txid(out)>-<n(out)>-<txid(in)>-<n(in)> = ts
// to sum up addr balance (only outs, spents are gotten later)
var ADDR_PREFIX = 'txa-'; //txa-<addr>-<txid>-<n>
// => + btc_sat:ts [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs]
var ADDR_PREFIX = 'txa2-'; //txa-<addr>-<tsr>-<txid>-<n>
// tsr = 1e13-js_timestamp
// => + btc_sat [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs]
// |balance:txApperances
// TODO: use bitcore networks module
var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
@ -17,6 +21,7 @@ var CONCURRENCY = 10;
var DEFAULT_SAFE_CONFIRMATIONS = 6;
var MAX_OPEN_FILES = 500;
var END_OF_WORLD_TS = 1e13;
// var CONFIRMATION_NR_TO_NOT_CHECK = 10; //Spend
/**
* Module dependencies.
@ -189,29 +194,19 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) {
return c_in(); // error not scalated
}
txInfo.firstSeenTs = ret.spentTs;
txInfo.firstSeenTs = ret.ts;
i.unconfirmedInput = i.unconfirmedInput;
i.addr = ret.addr;
i.valueSat = ret.valueSat;
i.value = ret.valueSat / util.COIN;
valueIn += i.valueSat;
/*
* If confirmed by bitcoind, we could not check for double spents
* but we prefer to keep the flag of double spent attempt
*
if (txInfo.confirmations
&& txInfo.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK)
return c_in();
isspent
*/
// Double spent?
if (ret.multipleSpentAttempt || !ret.spentTxId ||
(ret.spentTxId && ret.spentTxId !== txInfo.txid)
(ret.spentTxId && ret.spentTxId !== info.txid)
) {
if (ret.multipleSpentAttempts) {
ret.multipleSpentAttempts.forEach(function(mul) {
if (mul.spentTxId !== txInfo.txid) {
if (mul.spentTxId !== info.txid) {
i.doubleSpentTxID = ret.spentTxId;
i.doubleSpentIndex = ret.spentIndex;
}
@ -275,39 +270,27 @@ TransactionDb.prototype.fromIdWithInfo = function(txid, cb) {
});
};
// Gets address info from an outpoint
TransactionDb.prototype.fromTxIdN = function(txid, n, cb) {
var self = this;
var k = OUTS_PREFIX + txid + '-' + n;
db.get(k, function(err, val) {
var ret;
if (!val || (err && err.notFound)) {
return cb(null, {
unconfirmedInput: 1
});
err=null;
ret= { unconfirmedInput: 1 };
}
var a = val.split(':');
var ret = {
addr: a[0],
valueSat: parseInt(a[1]),
};
// spent?
var k = SPENT_PREFIX + txid + '-' + n + '-';
db.createReadStream({
start: k,
end: k + '~'
})
.on('data', function(data) {
var k = data.key.split('-');
self._addSpentInfo(ret, k[3], k[4], data.value);
})
.on('error', function(error) {
return cb(error);
})
.on('end', function() {
return cb(null, ret);
});
else {
var a = val.split(':');
ret = {
addr: a[0],
valueSat: parseInt(a[1]),
// ts: parseInt(a[2]), // TODO
};
}
return cb(err, ret);
});
};
@ -324,7 +307,7 @@ TransactionDb.prototype.deleteCacheForAddress = function(addr,cb) {
dbScript.push({
type: 'put',
key: data.key,
value: v.slice(0,2).join(':'),
value: v[0],
});
})
.on('error', function(err) {
@ -362,7 +345,7 @@ TransactionDb.prototype.cacheConfirmations = function(txouts,cb) {
//console.log('[TransactionDb.js.352:infoToCache:]',infoToCache); //TODO
if (infoToCache.length){
infoToCache.unshift(txout.value_sat,txout.ts);
infoToCache.unshift(txout.value_sat);
//console.log('[BlockDb.js.373:txs:]' ,txout.key, infoToCache.join(':')); //TODO
dbScript.push({
type: 'put',
@ -409,50 +392,51 @@ TransactionDb.prototype.cacheScriptPubKey = function(txouts,cb) {
TransactionDb.prototype._parseAddrData = function(data) {
var k = data.key.split('-');
var v = data.value.split(':');
// console.log('[TransactionDb.js.410]',v); //TODO
// console.log('[TransactionDb.js.375]',data.key,data.value); //TODO
var item = {
key: data.key,
txid: k[2],
index: parseInt(k[3]),
ts: parseInt(k[2]),
txid: k[3],
index: parseInt(k[4]),
value_sat: parseInt(v[0]),
ts: parseInt(v[1]),
};
// Cache:
// v[2]== isConfirmedCached
// v[3]=== '1' -> is SpendCached -> [4]=spendTxId [5]=spentIndex [6]=spendTs
// v[4]!== '1' -> is ScriptPubkey -> [[3] = scriptPubkey
if (v[2]){
// v[1]== isConfirmedCached
// v[2]=== '1' -> is SpendCached -> [4]=spendTxId [5]=spentIndex [6]=spendTs
// v[3]!== '1' -> is ScriptPubkey -> [[3] = scriptPubkey
if (v[1]){
item.isConfirmed = 1;
item.isConfirmedCached = 1;
// console.log('[TransactionDb.js.356] CACHE HIT CONF:', item.key); //TODO
// Sent, confirmed
if (v[3] === '1'){
if (v[2] === '1'){
// console.log('[TransactionDb.js.356] CACHE HIT SPENT:', item.key); //TODO
item.spentIsConfirmed = 1;
item.spentIsConfirmedCached = 1;
item.spentTxId = v[4];
item.spentIndex = parseInt(v[5]);
item.spentTs = parseInt(v[6]);
item.spentTxId = v[3];
item.spentIndex = parseInt(v[4]);
item.spentTs = parseInt(v[5]);
}
// Scriptpubkey cached
else if (v[3]) {
else if (v[2]) {
// console.log('[TransactionDb.js.356] CACHE HIT SCRIPTPUBKEY:', item.key); //TODO
item.scriptPubKey = v[3];
item.scriptPubKey = v[2];
item.scriptPubKeyCached = 1;
}
}
return item;
};
TransactionDb.prototype.fromAddr = function(addr, cb) {
TransactionDb.prototype.fromAddr = function(addr, cb, txLimit) {
var self = this;
var k = ADDR_PREFIX + addr + '-';
var ret = [];
db.createReadStream({
start: k,
end: k + '~'
end: k + '~',
limit: txLimit>0 ? txLimit: -1, // -1 means not limit
})
.on('data', function(data) {
ret.push(self._parseAddrData(data));
@ -597,14 +581,15 @@ TransactionDb.prototype._addScript = function(tx, relatedAddrs) {
if (relatedAddrs) relatedAddrs[addr]=1;
var k = OUTS_PREFIX + txid + '-' + o.n;
var tsr = END_OF_WORLD_TS - ts;
dbScript.push({
type: 'put',
key: k,
value: addr + ':' + sat,
},{
type: 'put',
key: ADDR_PREFIX + addr + '-' + txid + '-' + o.n,
value: sat + ':' + ts,
key: ADDR_PREFIX + addr + '-' + tsr + '-'+ txid + '-' + o.n,
value: sat,
});
}
}
@ -673,12 +658,55 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) {
TransactionDb.prototype.checkVersion02 = function(cb) {
var k = 'txb-f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba-00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206';
db.get(k, function(err, val) {
var k = 'txa-';
var isV2=1;
db.createReadStream({
start: k,
end: k + '~',
limit: 1,
})
.on('data', function(data) {
isV2=0;
})
.on('end', function (){
return cb(isV2);
});
};
return cb(!val);
});
TransactionDb.prototype.migrateV02 = function(cb) {
var k = 'txa-';
var dbScript = [];
var c=0;
var c2=0;
var N=50000;
db.createReadStream({
start: k,
end: k + '~'
})
.on('data', function(data) {
var k = data.key.split('-');
var v = data.value.split(':');
dbScript.push({
type: 'put',
key: ADDR_PREFIX + k[1] + '-' + (END_OF_WORLD_TS - parseInt(v[1])) + '-' + k[3] + '-' + [4],
value: v[0],
});
if (c++>N) {
console.log('\t%dM txs processed', ((c2+=N)/1e6).toFixed(3)); //TODO
db.batch(dbScript,function () {
c=0;
dbScript=[];
});
}
})
.on('error', function(err) {
return cb(err);
})
.on('end', function (){
return cb();
});
};
module.exports = require('soop')(TransactionDb);

15
test/integration/addr.js
View File

@ -19,7 +19,14 @@ describe('Address balances', function() {
before(function(c) {
txDb = TransactionDb;
return c();
var l =addrValid.length;
var i =0;
addrValid.forEach(function(v) {
TransactionDb.deleteCacheForAddress(v.addr, function() {
if (++i===l) return c();
});
});
});
addrValid.forEach(function(v) {
@ -47,7 +54,7 @@ describe('Address balances', function() {
if (v.transactions) {
v.transactions.forEach(function(tx) {
assert(a.transactions.indexOf(tx) > -1, 'have tx ' + tx);
a.transactions.should.include(tx);
});
}
done();
@ -59,7 +66,7 @@ describe('Address balances', function() {
a.update(function(err) {
if (err) done(err);
v.addr.should.equal(a.addrStr);
a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances');
a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances');
a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat');
if (v.txApperances)
a.txApperances.should.equal(v.txApperances, 'txApperances');
@ -68,7 +75,7 @@ describe('Address balances', function() {
if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent);
if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance);
done();
},{noTxList:1});
},{txLimit:0});
});
}
});

8
util/updateToV0.2.js → util/upgradeV0.2js
View File

@ -22,6 +22,10 @@ async.series([
return c(err);
});
},
function(c){
console.log('[1/3] Migrating txs ... (this will take some minutes...)'); //TODO
txDb.migrateV02(c);
},
function(c){
var script=[];
async.whilst(
@ -36,7 +40,7 @@ async.series([
hash = val;
if (hash) height++;
if (!(height%1000) || !hash) {
console.log('*update 1/2\t%d blocks processed', height);
console.log('[2/3] migrating blocks \t%d blocks processed', height);
bDb._runScript(script, function(err) {
script=[];
return w_cb(err);
@ -47,7 +51,7 @@ async.series([
}, c);
},
function(c){
console.log('Migrating txs... (this will take some minutes...)'); //TODO
console.log('[3/3] Migrating txs... (this will take some minutes...)'); //TODO
bDb.migrateV02(c);
},
function(c){