From 8f1f1c87ace0287dfd967b320bfe931b32a299e2 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 31 Jan 2019 08:30:18 +0100 Subject: [PATCH] Modify the discovery of xpub addresses --- api/xpub.go | 144 ++++++++++++++++++------------------- server/public.go | 14 +++- static/templates/xpub.html | 4 +- 3 files changed, 83 insertions(+), 79 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index 6c44e4fa..5c35dc1a 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -13,7 +13,7 @@ import ( ) const xpubLen = 111 -const derivedAddressesBlock = 20 +const defaultAddressesGap = 20 var cachedXpubs = make(map[string]*xpubData) var cachedXpubsMux sync.Mutex @@ -31,6 +31,7 @@ type xpubAddress struct { } type xpubData struct { + gap int dataHeight uint32 dataHash string txs uint32 @@ -116,6 +117,50 @@ func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, e return false, nil } +func (w *Worker) scanAddresses(xpub string, data *xpubData, addresses []xpubAddress, gap int, change int, minDerivedIndex int, fork bool) (int, []xpubAddress, error) { + // rescan known addresses + lastUsed := 0 + for i := range addresses { + ad := &addresses[i] + if fork { + ad.bottomHeight = 0 + } + used, err := w.derivedAddressBalance(data, ad) + if err != nil { + return 0, nil, err + } + if used { + lastUsed = i + } + } + // derive new addresses as necessary + missing := len(addresses) - lastUsed + for missing < gap { + from := len(addresses) + to := from + gap - missing + if to < minDerivedIndex { + to = minDerivedIndex + } + descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, uint32(change), uint32(from), uint32(to)) + if err != nil { + return 0, nil, err + } + for i, a := range descriptors { + ad := xpubAddress{addrDesc: a} + used, err := w.derivedAddressBalance(data, &ad) + if err != nil { + return 0, nil, err + } + if used { + lastUsed = i + from + } + addresses = append(addresses, ad) + } + missing = len(addresses) - lastUsed + } + return lastUsed, addresses, nil +} + func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token { a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) var address string @@ -133,11 +178,16 @@ func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index in } // GetAddressForXpub computes address value and gets transactions for given address -func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { +func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*Address, error) { if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { return nil, ErrUnsupportedXpub } start := time.Now() + if gap <= 0 { + gap = defaultAddressesGap + } + // gap is increased one as there must be gap of empty addresses before the derivation is stopped + gap++ var processedHash string cachedXpubsMux.Lock() data, found := cachedXpubs[xpub] @@ -152,8 +202,8 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option break } fork := false - if !found { - data = &xpubData{} + if !found || data.gap != gap { + data = &xpubData{gap: gap} } else { hash, err := w.db.GetBlockHash(data.dataHeight) if err != nil { @@ -169,72 +219,14 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if data.dataHeight < bestheight { data.dataHeight = bestheight data.dataHash = besthash - // rescan known addresses - lastUsed := 0 - for i := range data.addresses { - ad := &data.addresses[i] - if fork { - ad.bottomHeight = 0 - } - used, err := w.derivedAddressBalance(data, ad) - if err != nil { - return nil, err - } - if used { - lastUsed = i - } + var lastUsedIndex int + lastUsedIndex, data.addresses, err = w.scanAddresses(xpub, data, data.addresses, gap, 0, 0, fork) + if err != nil { + return nil, err } - // derive new addresses as necessary - missing := len(data.addresses) - lastUsed - for missing < derivedAddressesBlock { - from := len(data.addresses) - descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, 0, uint32(from), uint32(from+derivedAddressesBlock-missing)) - if err != nil { - return nil, err - } - for i, a := range descriptors { - ad := xpubAddress{addrDesc: a} - used, err := w.derivedAddressBalance(data, &ad) - if err != nil { - return nil, err - } - if used { - lastUsed = i + from - } - data.addresses = append(data.addresses, ad) - } - missing = len(data.addresses) - lastUsed - } - // check and generate change addresses - ca := data.changeAddresses - data.changeAddresses = make([]xpubAddress, len(data.addresses)) - copy(data.changeAddresses, ca) - changeIndexes := []uint32{} - for i, ad := range data.addresses { - if ad.balance != nil { - if data.changeAddresses[i].addrDesc == nil { - changeIndexes = append(changeIndexes, uint32(i)) - } else { - _, err := w.derivedAddressBalance(data, &ad) - if err != nil { - return nil, err - } - } - } - } - if len(changeIndexes) > 0 { - descriptors, err := w.chainParser.DeriveAddressDescriptors(xpub, 1, changeIndexes) - if err != nil { - return nil, err - } - for i, a := range descriptors { - ad := &data.changeAddresses[changeIndexes[i]] - ad.addrDesc = a - _, err := w.derivedAddressBalance(data, ad) - if err != nil { - return nil, err - } - } + _, data.changeAddresses, err = w.scanAddresses(xpub, data, data.changeAddresses, gap, 1, lastUsedIndex, fork) + if err != nil { + return nil, err } } } @@ -242,12 +234,16 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubs[xpub] = data cachedXpubsMux.Unlock() tokens := make([]Token, 0, 4) - for i, ad := range data.addresses { + for i := range data.addresses { + ad := &data.addresses[i] if ad.balance != nil { - tokens = append(tokens, w.tokenFromXpubAddress(&ad, 0, i)) + tokens = append(tokens, w.tokenFromXpubAddress(ad, 0, i)) } - if data.changeAddresses[i].balance != nil { - tokens = append(tokens, w.tokenFromXpubAddress(&data.changeAddresses[i], 1, i)) + } + for i := range data.changeAddresses { + ad := &data.changeAddresses[i] + if ad.balance != nil { + tokens = append(tokens, w.tokenFromXpubAddress(ad, 1, i)) } } var totalReceived big.Int @@ -267,6 +263,6 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option // Erc20Contract: erc20c, // Nonce: nonce, } - glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) + glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) return &addr, nil } diff --git a/server/public.go b/server/public.go index 02c79120..a993cf63 100644 --- a/server/public.go +++ b/server/public.go @@ -607,7 +607,11 @@ func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl } } } - address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}) + gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) + if ec != nil { + gap = 0 + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}, gap) if err != nil { return errorTpl, nil, err } @@ -685,7 +689,7 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t var err error s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() if len(q) > 0 { - address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, 0) if err == nil { http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302) return noTpl, nil, nil @@ -879,7 +883,11 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er if ec != nil { page = 0 } - address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) + if ec != nil { + gap = 0 + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, gap) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 7dbebf46..94b9a283 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -34,8 +34,8 @@ Address Balance - Txs - Path + Txs + Path {{- range $t := $addr.Tokens -}}