diff --git a/api/types.go b/api/types.go index cd1c7450..add7f7fd 100644 --- a/api/types.go +++ b/api/types.go @@ -5,6 +5,7 @@ import ( "blockbook/common" "blockbook/db" "encoding/json" + "errors" "math/big" "time" ) @@ -27,6 +28,9 @@ const ( TxHistory ) +// ErrUnsupportedXpub is returned when coin type does not support xpub address derivation or provided string is not an xpub +var ErrUnsupportedXpub = errors.New("XPUB not supported") + // APIError extends error by information if the error details should be returned to the end user type APIError struct { Text string @@ -123,6 +127,9 @@ type TokenType string // ERC20TokenType is Ethereum ERC20 token const ERC20TokenType TokenType = "ERC20" +// XPUBAddressTokenType is address derived from xpub +const XPUBAddressTokenType TokenType = "XPUBAddress" + // Token contains info about tokens held by an address type Token struct { Type TokenType `json:"type"` diff --git a/api/xpub.go b/api/xpub.go new file mode 100644 index 00000000..6c44e4fa --- /dev/null +++ b/api/xpub.go @@ -0,0 +1,272 @@ +package api + +import ( + "blockbook/bchain" + "blockbook/db" + "fmt" + "math/big" + "sync" + "time" + + "github.com/golang/glog" + "github.com/juju/errors" +) + +const xpubLen = 111 +const derivedAddressesBlock = 20 + +var cachedXpubs = make(map[string]*xpubData) +var cachedXpubsMux sync.Mutex + +type txHeight struct { + txid string + height uint32 + addrIndex uint32 +} + +type xpubAddress struct { + addrDesc bchain.AddressDescriptor + balance *db.AddrBalance + bottomHeight uint32 +} + +type xpubData struct { + dataHeight uint32 + dataHash string + txs uint32 + sentSat big.Int + balanceSat big.Int + addresses []xpubAddress + changeAddresses []xpubAddress + txids []txHeight +} + +func (w *Worker) getAddressTxHeights(addrDesc bchain.AddressDescriptor, addrIndex uint32, mempool bool, filter *AddressFilter, maxResults int) ([]txHeight, error) { + var err error + txHeights := make([]txHeight, 0, 4) + var callback db.GetTransactionsCallback + if filter.Vout == AddressFilterVoutOff { + callback = func(txid string, height uint32, indexes []int32) error { + txHeights = append(txHeights, txHeight{txid, height, addrIndex}) + // take all txs in the last found block even if it exceeds maxResults + if len(txHeights) >= maxResults && txHeights[len(txHeights)-1].height != height { + return &db.StopIteration{} + } + return nil + } + } else { + callback = func(txid string, height uint32, indexes []int32) error { + for _, index := range indexes { + vout := index + if vout < 0 { + vout = ^vout + } + if (filter.Vout == AddressFilterVoutInputs && index < 0) || + (filter.Vout == AddressFilterVoutOutputs && index >= 0) || + (vout == int32(filter.Vout)) { + txHeights = append(txHeights, txHeight{txid, height, addrIndex}) + if len(txHeights) >= maxResults { + return &db.StopIteration{} + } + break + } + } + return nil + } + } + if mempool { + uniqueTxs := make(map[string]struct{}) + o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) + if err != nil { + return nil, err + } + for _, m := range o { + if _, found := uniqueTxs[m.Txid]; !found { + l := len(txHeights) + callback(m.Txid, 0, []int32{m.Vout}) + if len(txHeights) > l { + uniqueTxs[m.Txid] = struct{}{} + } + } + } + } else { + to := filter.ToHeight + if to == 0 { + to = ^uint32(0) + } + err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback) + if err != nil { + return nil, err + } + } + return txHeights, nil +} + +func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, error) { + var err error + if ad.balance, err = w.db.GetAddrDescBalance(ad.addrDesc); err != nil { + return false, err + } + if ad.balance != nil { + data.txs += ad.balance.Txs + data.sentSat.Add(&data.sentSat, &ad.balance.SentSat) + data.balanceSat.Add(&data.balanceSat, &ad.balance.BalanceSat) + return true, nil + } + return false, nil +} + +func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token { + a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) + var address string + if len(a) > 0 { + address = a[0] + } + return Token{ + Type: XPUBAddressTokenType, + Name: address, + Decimals: w.chainParser.AmountDecimals(), + BalanceSat: (*Amount)(&ad.balance.BalanceSat), + Transfers: int(ad.balance.Txs), + Contract: fmt.Sprintf("%d/%d", changeIndex, index), + } +} + +// GetAddressForXpub computes address value and gets transactions for given address +func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { + if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { + return nil, ErrUnsupportedXpub + } + start := time.Now() + var processedHash string + cachedXpubsMux.Lock() + data, found := cachedXpubs[xpub] + cachedXpubsMux.Unlock() + // to load all data for xpub may take some time, perform it in a loop to process a possible new block + for { + bestheight, besthash, err := w.db.GetBestBlock() + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + if besthash == processedHash { + break + } + fork := false + if !found { + data = &xpubData{} + } else { + hash, err := w.db.GetBlockHash(data.dataHeight) + if err != nil { + return nil, err + } + if hash != data.dataHash { + // in case of for reset all cached txids + fork = true + data.txids = nil + } + } + processedHash = besthash + if data.dataHeight < bestheight { + data.dataHeight = bestheight + data.dataHash = besthash + // rescan known addresses + lastUsed := 0 + for i := range data.addresses { + ad := &data.addresses[i] + if fork { + ad.bottomHeight = 0 + } + used, err := w.derivedAddressBalance(data, ad) + if err != nil { + return nil, err + } + if used { + lastUsed = i + } + } + // derive new addresses as necessary + missing := len(data.addresses) - lastUsed + for missing < derivedAddressesBlock { + from := len(data.addresses) + descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, 0, uint32(from), uint32(from+derivedAddressesBlock-missing)) + if err != nil { + return nil, err + } + for i, a := range descriptors { + ad := xpubAddress{addrDesc: a} + used, err := w.derivedAddressBalance(data, &ad) + if err != nil { + return nil, err + } + if used { + lastUsed = i + from + } + data.addresses = append(data.addresses, ad) + } + missing = len(data.addresses) - lastUsed + } + // check and generate change addresses + ca := data.changeAddresses + data.changeAddresses = make([]xpubAddress, len(data.addresses)) + copy(data.changeAddresses, ca) + changeIndexes := []uint32{} + for i, ad := range data.addresses { + if ad.balance != nil { + if data.changeAddresses[i].addrDesc == nil { + changeIndexes = append(changeIndexes, uint32(i)) + } else { + _, err := w.derivedAddressBalance(data, &ad) + if err != nil { + return nil, err + } + } + } + } + if len(changeIndexes) > 0 { + descriptors, err := w.chainParser.DeriveAddressDescriptors(xpub, 1, changeIndexes) + if err != nil { + return nil, err + } + for i, a := range descriptors { + ad := &data.changeAddresses[changeIndexes[i]] + ad.addrDesc = a + _, err := w.derivedAddressBalance(data, ad) + if err != nil { + return nil, err + } + } + } + } + } + cachedXpubsMux.Lock() + cachedXpubs[xpub] = data + cachedXpubsMux.Unlock() + tokens := make([]Token, 0, 4) + for i, ad := range data.addresses { + if ad.balance != nil { + tokens = append(tokens, w.tokenFromXpubAddress(&ad, 0, i)) + } + if data.changeAddresses[i].balance != nil { + tokens = append(tokens, w.tokenFromXpubAddress(&data.changeAddresses[i], 1, i)) + } + } + var totalReceived big.Int + totalReceived.Add(&data.balanceSat, &data.sentSat) + addr := Address{ + // Paging: pg, + AddrStr: xpub, + BalanceSat: (*Amount)(&data.balanceSat), + TotalReceivedSat: (*Amount)(&totalReceived), + TotalSentSat: (*Amount)(&data.sentSat), + Txs: int(data.txs), + // UnconfirmedBalanceSat: (*Amount)(&uBalSat), + // UnconfirmedTxs: len(txm), + // Transactions: txs, + // Txids: txids, + Tokens: tokens, + // Erc20Contract: erc20c, + // Nonce: nonce, + } + glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) + return &addr, nil +} diff --git a/server/public.go b/server/public.go index a18b3efd..02c79120 100644 --- a/server/public.go +++ b/server/public.go @@ -131,6 +131,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { // internal explorer handlers serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx)) serveMux.HandleFunc(path+"address/", s.htmlTemplateHandler(s.explorerAddress)) + serveMux.HandleFunc(path+"xpub/", s.htmlTemplateHandler(s.explorerXpub)) serveMux.HandleFunc(path+"search/", s.htmlTemplateHandler(s.explorerSearch)) serveMux.HandleFunc(path+"blocks", s.htmlTemplateHandler(s.explorerBlocks)) serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock)) @@ -165,6 +166,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiDefault)) serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx, apiDefault)) serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress, apiDefault)) + serveMux.HandleFunc(path+"api/xpub/", s.jsonHandler(s.apiXpub, apiDefault)) serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo, apiDefault)) serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock, apiDefault)) serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx, apiDefault)) @@ -174,6 +176,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/v2/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV2)) serveMux.HandleFunc(path+"api/v2/tx/", s.jsonHandler(s.apiTx, apiV2)) serveMux.HandleFunc(path+"api/v2/address/", s.jsonHandler(s.apiAddress, apiV2)) + serveMux.HandleFunc(path+"api/v2/xpub/", s.jsonHandler(s.apiXpub, apiV2)) serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV2)) serveMux.HandleFunc(path+"api/v2/block/", s.jsonHandler(s.apiBlock, apiV2)) serveMux.HandleFunc(path+"api/v2/sendtx/", s.jsonHandler(s.apiSendTx, apiV2)) @@ -372,6 +375,7 @@ const ( indexTpl txTpl addressTpl + xpubTpl blocksTpl blockTpl sendTransactionTpl @@ -465,6 +469,7 @@ func (s *PublicServer) parseTemplates() []*template.Template { t[addressTpl] = createTemplate("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") t[blockTpl] = createTemplate("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") } + t[xpubTpl] = createTemplate("./static/templates/xpub.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") return t } @@ -577,6 +582,48 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( return addressTpl, data, nil } +func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var address *api.Address + var filter string + var fn = api.AddressFilterVoutOff + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + filter = r.URL.Query().Get("filter") + if len(filter) > 0 { + if filter == "inputs" { + fn = api.AddressFilterVoutInputs + } else if filter == "outputs" { + fn = api.AddressFilterVoutOutputs + } else { + fn, ec = strconv.Atoi(filter) + if ec != nil || fn < 0 { + filter = "" + fn = api.AddressFilterVoutOff + } + } + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}) + if err != nil { + return errorTpl, nil, err + } + } + data := s.newTemplateData() + data.AddrStr = address.AddrStr + data.Address = address + data.Page = address.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + if filter != "" { + data.PageParams = template.URL("&filter=" + filter) + data.Address.Filter = filter + } + return xpubTpl, data, nil +} + func (s *PublicServer) explorerBlocks(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var blocks *api.Blocks var err error @@ -638,6 +685,11 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t var err error s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() if len(q) > 0 { + address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + if err == nil { + http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302) + return noTpl, nil, nil + } block, err = s.api.GetBlock(q, 0, 1) if err == nil { http.Redirect(w, r, joinURL("/block/", block.Hash), 302) @@ -818,6 +870,23 @@ func (s *PublicServer) apiAddress(r *http.Request, apiVersion int) (interface{}, return address, err } +func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, error) { + var address *api.Address + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + if err == nil && apiVersion == apiV1 { + return s.api.AddressToV1(address), nil + } + } + return address, err +} + func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interface{}, error) { var utxo []api.AddressUtxo var err error diff --git a/static/templates/base.html b/static/templates/base.html index bc1e9f02..cecc88d9 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -42,7 +42,7 @@ {{- end -}} diff --git a/static/templates/xpub.html b/static/templates/xpub.html new file mode 100644 index 00000000..7dbebf46 --- /dev/null +++ b/static/templates/xpub.html @@ -0,0 +1,102 @@ +{{define "specific"}}{{$cs := .CoinShortcut}}{{$addr := .Address}}{{$data := .}} +

XPUB {{formatAmount $addr.BalanceSat}} {{$cs}} +

+
+ {{$addr.AddrStr}} +
+

Confirmed

+
+
+ + + + + + + + + + + + + + + + + + + {{- if $addr.Tokens -}} + + + + + {{- end -}} + +
Total Received{{formatAmount $addr.TotalReceivedSat}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSentSat}} {{$cs}}
Final Balance{{formatAmount $addr.BalanceSat}} {{$cs}}
No. Transactions{{$addr.Txs}}
XPUB addresses + + + + + + + + + {{- range $t := $addr.Tokens -}} + + + + + + + {{- end -}} + +
AddressBalanceTxsPath
{{$t.Name}}{{formatAmount $t.BalanceSat}} {{$cs}}{{$t.Transfers}}{{$t.Contract}}
+
+
+
+
+ + +
+
+{{- if $addr.UnconfirmedTxs -}} +

Unconfirmed

+
+ + + + + + + + + + + +
Unconfirmed Balance{{formatAmount $addr.UnconfirmedBalanceSat}} {{$cs}}
No. Transactions{{$addr.UnconfirmedTxs}}
+
+{{- end}}{{if or $addr.Transactions $addr.Filter -}} +
+

Transactions

+ +
+ +
+
+
+ {{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}} +
+ +{{end}}{{end}} \ No newline at end of file