RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Increase xpub cache, reduce logging

Browse Source
This commit is contained in:
Martin Boehm 2021-04-26 20:06:51 +02:00
parent 9c44fd4f22
commit 66023becd6
2 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
api/worker.go
View File

@ -105,7 +105,7 @@ func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
glog.Info("GetSpendingTxid ", txid, " ", n, " finished in ", time.Since(start)) glog.Info("GetSpendingTxid ", txid, " ", n, ", ", time.Since(start))
return tx.Vout[n].SpentTxID, nil return tx.Vout[n].SpentTxID, nil
} }
@ -954,7 +954,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option Acco
Erc20Contract: erc20c, Erc20Contract: erc20c,
Nonce: nonce, Nonce: nonce,
} }
glog.Info("GetAddress ", address, " finished in ", time.Since(start)) glog.Info("GetAddress ", address, ", ", time.Since(start))
return r, nil return r, nil
} }
@ -1157,7 +1157,7 @@ func (w *Worker) GetBalanceHistory(address string, fromTimestamp, toTimestamp in
if err != nil { if err != nil {
return nil, err return nil, err
} }
glog.Info("GetBalanceHistory ", address, ", blocks ", fromHeight, "-", toHeight, ", count ", len(bha), " finished in ", time.Since(start)) glog.Info("GetBalanceHistory ", address, ", blocks ", fromHeight, "-", toHeight, ", count ", len(bha), ", ", time.Since(start))
return bha, nil return bha, nil
} }
@ -1305,7 +1305,7 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) (Utxos, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
glog.Info("GetAddressUtxo ", address, ", ", len(r), " utxos, finished in ", time.Since(start)) glog.Info("GetAddressUtxo ", address, ", ", len(r), " utxos, ", time.Since(start))
return r, nil return r, nil
} }
@ -1335,7 +1335,7 @@ func (w *Worker) GetBlocks(page int, blocksOnPage int) (*Blocks, error) {
} }
r.Blocks[i-from] = *bi r.Blocks[i-from] = *bi
} }
glog.Info("GetBlocks page ", page, " finished in ", time.Since(start)) glog.Info("GetBlocks page ", page, ", ", time.Since(start))
return r, nil return r, nil
} }
@ -1597,7 +1597,7 @@ func (w *Worker) GetFeeStats(bid string) (*FeeStats, error) {
} }
} }
glog.Info("GetFeeStats ", bid, " (", len(feesPerKb), " txs) finished in ", time.Since(start)) glog.Info("GetFeeStats ", bid, " (", len(feesPerKb), " txs), ", time.Since(start))
return &FeeStats{ return &FeeStats{
TxCount: len(feesPerKb), TxCount: len(feesPerKb),
@ -1649,7 +1649,7 @@ func (w *Worker) GetBlock(bid string, page int, txsOnPage int) (*Block, error) {
} }
txs = txs[:txi] txs = txs[:txi]
bi.Txids = nil bi.Txids = nil
glog.Info("GetBlock ", bid, ", page ", page, " finished in ", time.Since(start)) glog.Info("GetBlock ", bid, ", page ", page, ", ", time.Since(start))
return &Block{ return &Block{
Paging: pg, Paging: pg,
BlockInfo: BlockInfo{ BlockInfo: BlockInfo{
@ -1790,7 +1790,7 @@ func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) {
Consensus: ci.Consensus, Consensus: ci.Consensus,
} }
w.is.SetBackendInfo(backendInfo) w.is.SetBackendInfo(backendInfo)
glog.Info("GetSystemInfo finished in ", time.Since(start)) glog.Info("GetSystemInfo, ", time.Since(start))
return &SystemInfo{blockbookInfo, backendInfo}, nil return &SystemInfo{blockbookInfo, backendInfo}, nil
} }

40
api/xpub.go
View File

@ -19,8 +19,8 @@ const maxAddressesGap = 10000
const txInput = 1 const txInput = 1
const txOutput = 2 const txOutput = 2
const xpubCacheSize = 512 const xpubCacheSize = 4096
const xpubCacheExpirationSeconds = 7200 const xpubCacheExpirationSeconds = 3600
var cachedXpubs = make(map[string]xpubData) var cachedXpubs = make(map[string]xpubData)
var cachedXpubsMux sync.Mutex var cachedXpubsMux sync.Mutex
@ -266,12 +266,12 @@ func evictXpubCacheItems() {
delete(cachedXpubs, oldestKey) delete(cachedXpubs, oldestKey)
count++ count++
} }
glog.Info("Evicted ", count, " items from xpub cache, oldest item accessed at ", time.Unix(oldest, 0), ", cache size ", len(cachedXpubs)) glog.V(1).Info("Evicted ", count, " items from xpub cache, oldest item accessed at ", time.Unix(oldest, 0), ", cache size ", len(cachedXpubs))
} }
func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option AccountDetails, filter *AddressFilter, gap int) (*xpubData, uint32, error) { func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option AccountDetails, filter *AddressFilter, gap int) (*xpubData, uint32, bool, error) {
if w.chainType != bchain.ChainBitcoinType { if w.chainType != bchain.ChainBitcoinType {
return nil, 0, ErrUnsupportedXpub return nil, 0, false, ErrUnsupportedXpub
} }
var ( var (
err error err error
@ -288,28 +288,28 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option Accoun
gap++ gap++
var processedHash string var processedHash string
cachedXpubsMux.Lock() cachedXpubsMux.Lock()
data, found := cachedXpubs[xpub] data, inCache := cachedXpubs[xpub]
cachedXpubsMux.Unlock() cachedXpubsMux.Unlock()
// to load all data for xpub may take some time, do it in a loop to process a possible new block // to load all data for xpub may take some time, do it in a loop to process a possible new block
for { for {
bestheight, besthash, err = w.db.GetBestBlock() bestheight, besthash, err = w.db.GetBestBlock()
if err != nil { if err != nil {
return nil, 0, errors.Annotatef(err, "GetBestBlock") return nil, 0, inCache, errors.Annotatef(err, "GetBestBlock")
} }
if besthash == processedHash { if besthash == processedHash {
break break
} }
fork := false fork := false
if !found || data.gap != gap { if !inCache || data.gap != gap {
data = xpubData{gap: gap} data = xpubData{gap: gap}
data.basePath, err = w.chainParser.DerivationBasePath(xpub) data.basePath, err = w.chainParser.DerivationBasePath(xpub)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, inCache, err
} }
} else { } else {
hash, err := w.db.GetBlockHash(data.dataHeight) hash, err := w.db.GetBlockHash(data.dataHeight)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, inCache, err
} }
if hash != data.dataHash { if hash != data.dataHash {
// in case of for reset all cached data // in case of for reset all cached data
@ -326,18 +326,18 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option Accoun
var lastUsedIndex int var lastUsedIndex int
lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, &data, data.addresses, gap, 0, 0, fork) lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, &data, data.addresses, gap, 0, 0, fork)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, inCache, err
} }
_, data.changeAddresses, err = w.xpubScanAddresses(xpub, &data, data.changeAddresses, gap, 1, lastUsedIndex, fork) _, data.changeAddresses, err = w.xpubScanAddresses(xpub, &data, data.changeAddresses, gap, 1, lastUsedIndex, fork)
if err != nil { if err != nil {
return nil, 0, err return nil, 0, inCache, err
} }
} }
if option >= AccountDetailsTxidHistory { if option >= AccountDetailsTxidHistory {
for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} {
for i := range da { for i := range da {
if err = w.xpubCheckAndLoadTxids(&da[i], filter, bestheight, (page+1)*txsOnPage); err != nil { if err = w.xpubCheckAndLoadTxids(&da[i], filter, bestheight, (page+1)*txsOnPage); err != nil {
return nil, 0, err return nil, 0, inCache, err
} }
} }
} }
@ -350,7 +350,7 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option Accoun
} }
cachedXpubs[xpub] = data cachedXpubs[xpub] = data
cachedXpubsMux.Unlock() cachedXpubsMux.Unlock()
return &data, bestheight, nil return &data, bestheight, inCache, nil
} }
// GetXpubAddress computes address value and gets transactions for given address // GetXpubAddress computes address value and gets transactions for given address
@ -376,7 +376,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc
uBalSat big.Int uBalSat big.Int
unconfirmedTxs int unconfirmedTxs int
) )
data, bestheight, err := w.getXpubData(xpub, page, txsOnPage, option, filter, gap) data, bestheight, inCache, err := w.getXpubData(xpub, page, txsOnPage, option, filter, gap)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -544,14 +544,14 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc
Tokens: tokens, Tokens: tokens,
XPubAddresses: xpubAddresses, XPubAddresses: xpubAddresses,
} }
glog.Info("GetXpubAddress ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", txCount, " confirmed txs, finished in ", time.Since(start)) glog.Info("GetXpubAddress ", xpub[:16], ", cache ", inCache, ", ", len(data.addresses)+len(data.changeAddresses), " addresses, ", txCount, " txs, ", time.Since(start))
return &addr, nil return &addr, nil
} }
// GetXpubUtxo returns unspent outputs for given xpub // GetXpubUtxo returns unspent outputs for given xpub
func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, error) { func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, error) {
start := time.Now() start := time.Now()
data, _, err := w.getXpubData(xpub, 0, 1, AccountDetailsBasic, &AddressFilter{ data, _, inCache, err := w.getXpubData(xpub, 0, 1, AccountDetailsBasic, &AddressFilter{
Vout: AddressFilterVoutOff, Vout: AddressFilterVoutOff,
OnlyConfirmed: onlyConfirmed, OnlyConfirmed: onlyConfirmed,
}, gap) }, gap)
@ -585,7 +585,7 @@ func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, e
} }
} }
sort.Stable(r) sort.Stable(r)
glog.Info("GetXpubUtxo ", xpub[:16], ", ", len(r), " utxos, finished in ", time.Since(start)) glog.Info("GetXpubUtxo ", xpub[:16], ", cache ", inCache, ", ", len(r), " utxos, ", time.Since(start))
return r, nil return r, nil
} }
@ -597,7 +597,7 @@ func (w *Worker) GetXpubBalanceHistory(xpub string, fromTimestamp, toTimestamp i
if fromHeight >= toHeight { if fromHeight >= toHeight {
return bhs, nil return bhs, nil
} }
data, _, err := w.getXpubData(xpub, 0, 1, AccountDetailsTxidHistory, &AddressFilter{ data, _, inCache, err := w.getXpubData(xpub, 0, 1, AccountDetailsTxidHistory, &AddressFilter{
Vout: AddressFilterVoutOff, Vout: AddressFilterVoutOff,
OnlyConfirmed: true, OnlyConfirmed: true,
FromHeight: fromHeight, FromHeight: fromHeight,
@ -632,6 +632,6 @@ func (w *Worker) GetXpubBalanceHistory(xpub string, fromTimestamp, toTimestamp i
if err != nil { if err != nil {
return nil, err return nil, err
} }
glog.Info("GetUtxoBalanceHistory ", xpub[:16], ", blocks ", fromHeight, "-", toHeight, ", count ", len(bha), ", finished in ", time.Since(start)) glog.Info("GetUtxoBalanceHistory ", xpub[:16], ", cache ", inCache, ", blocks ", fromHeight, "-", toHeight, ", count ", len(bha), ", ", time.Since(start))
return bha, nil return bha, nil
} }