Implement GetAddressForXpub and xpub explorer view - WIP

api/types.go

@@ -5,6 +5,7 @@ import (
 	"blockbook/common"
 	"blockbook/db"
 	"encoding/json"
+	"errors"
 	"math/big"
 	"time"
 )
@@ -27,6 +28,9 @@ const (
 	TxHistory
 )
 
+// ErrUnsupportedXpub is returned when coin type does not support xpub address derivation or provided string is not an xpub
+var ErrUnsupportedXpub = errors.New("XPUB not supported")
+
 // APIError extends error by information if the error details should be returned to the end user
 type APIError struct {
 	Text   string
@@ -123,6 +127,9 @@ type TokenType string
 // ERC20TokenType is Ethereum ERC20 token
 const ERC20TokenType TokenType = "ERC20"
 
+// XPUBAddressTokenType is address derived from xpub
+const XPUBAddressTokenType TokenType = "XPUBAddress"
+
 // Token contains info about tokens held by an address
 type Token struct {
 	Type          TokenType `json:"type"`

api/xpub.go

@@ -0,0 +1,272 @@
+package api
+
+import (
+	"blockbook/bchain"
+	"blockbook/db"
+	"fmt"
+	"math/big"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/juju/errors"
+)
+
+const xpubLen = 111
+const derivedAddressesBlock = 20
+
+var cachedXpubs = make(map[string]*xpubData)
+var cachedXpubsMux sync.Mutex
+
+type txHeight struct {
+	txid      string
+	height    uint32
+	addrIndex uint32
+}
+
+type xpubAddress struct {
+	addrDesc     bchain.AddressDescriptor
+	balance      *db.AddrBalance
+	bottomHeight uint32
+}
+
+type xpubData struct {
+	dataHeight      uint32
+	dataHash        string
+	txs             uint32
+	sentSat         big.Int
+	balanceSat      big.Int
+	addresses       []xpubAddress
+	changeAddresses []xpubAddress
+	txids           []txHeight
+}
+
+func (w *Worker) getAddressTxHeights(addrDesc bchain.AddressDescriptor, addrIndex uint32, mempool bool, filter *AddressFilter, maxResults int) ([]txHeight, error) {
+	var err error
+	txHeights := make([]txHeight, 0, 4)
+	var callback db.GetTransactionsCallback
+	if filter.Vout == AddressFilterVoutOff {
+		callback = func(txid string, height uint32, indexes []int32) error {
+			txHeights = append(txHeights, txHeight{txid, height, addrIndex})
+			// take all txs in the last found block even if it exceeds maxResults
+			if len(txHeights) >= maxResults && txHeights[len(txHeights)-1].height != height {
+				return &db.StopIteration{}
+			}
+			return nil
+		}
+	} else {
+		callback = func(txid string, height uint32, indexes []int32) error {
+			for _, index := range indexes {
+				vout := index
+				if vout < 0 {
+					vout = ^vout
+				}
+				if (filter.Vout == AddressFilterVoutInputs && index < 0) ||
+					(filter.Vout == AddressFilterVoutOutputs && index >= 0) ||
+					(vout == int32(filter.Vout)) {
+					txHeights = append(txHeights, txHeight{txid, height, addrIndex})
+					if len(txHeights) >= maxResults {
+						return &db.StopIteration{}
+					}
+					break
+				}
+			}
+			return nil
+		}
+	}
+	if mempool {
+		uniqueTxs := make(map[string]struct{})
+		o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc)
+		if err != nil {
+			return nil, err
+		}
+		for _, m := range o {
+			if _, found := uniqueTxs[m.Txid]; !found {
+				l := len(txHeights)
+				callback(m.Txid, 0, []int32{m.Vout})
+				if len(txHeights) > l {
+					uniqueTxs[m.Txid] = struct{}{}
+				}
+			}
+		}
+	} else {
+		to := filter.ToHeight
+		if to == 0 {
+			to = ^uint32(0)
+		}
+		err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return txHeights, nil
+}
+
+func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, error) {
+	var err error
+	if ad.balance, err = w.db.GetAddrDescBalance(ad.addrDesc); err != nil {
+		return false, err
+	}
+	if ad.balance != nil {
+		data.txs += ad.balance.Txs
+		data.sentSat.Add(&data.sentSat, &ad.balance.SentSat)
+		data.balanceSat.Add(&data.balanceSat, &ad.balance.BalanceSat)
+		return true, nil
+	}
+	return false, nil
+}
+
+func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token {
+	a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc)
+	var address string
+	if len(a) > 0 {
+		address = a[0]
+	}
+	return Token{
+		Type:       XPUBAddressTokenType,
+		Name:       address,
+		Decimals:   w.chainParser.AmountDecimals(),
+		BalanceSat: (*Amount)(&ad.balance.BalanceSat),
+		Transfers:  int(ad.balance.Txs),
+		Contract:   fmt.Sprintf("%d/%d", changeIndex, index),
+	}
+}
+
+// GetAddressForXpub computes address value and gets transactions for given address
+func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) {
+	if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen {
+		return nil, ErrUnsupportedXpub
+	}
+	start := time.Now()
+	var processedHash string
+	cachedXpubsMux.Lock()
+	data, found := cachedXpubs[xpub]
+	cachedXpubsMux.Unlock()
+	// to load all data for xpub may take some time, perform it in a loop to process a possible new block
+	for {
+		bestheight, besthash, err := w.db.GetBestBlock()
+		if err != nil {
+			return nil, errors.Annotatef(err, "GetBestBlock")
+		}
+		if besthash == processedHash {
+			break
+		}
+		fork := false
+		if !found {
+			data = &xpubData{}
+		} else {
+			hash, err := w.db.GetBlockHash(data.dataHeight)
+			if err != nil {
+				return nil, err
+			}
+			if hash != data.dataHash {
+				// in case of for reset all cached txids
+				fork = true
+				data.txids = nil
+			}
+		}
+		processedHash = besthash
+		if data.dataHeight < bestheight {
+			data.dataHeight = bestheight
+			data.dataHash = besthash
+			// rescan known addresses
+			lastUsed := 0
+			for i := range data.addresses {
+				ad := &data.addresses[i]
+				if fork {
+					ad.bottomHeight = 0
+				}
+				used, err := w.derivedAddressBalance(data, ad)
+				if err != nil {
+					return nil, err
+				}
+				if used {
+					lastUsed = i
+				}
+			}
+			// derive new addresses as necessary
+			missing := len(data.addresses) - lastUsed
+			for missing < derivedAddressesBlock {
+				from := len(data.addresses)
+				descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, 0, uint32(from), uint32(from+derivedAddressesBlock-missing))
+				if err != nil {
+					return nil, err
+				}
+				for i, a := range descriptors {
+					ad := xpubAddress{addrDesc: a}
+					used, err := w.derivedAddressBalance(data, &ad)
+					if err != nil {
+						return nil, err
+					}
+					if used {
+						lastUsed = i + from
+					}
+					data.addresses = append(data.addresses, ad)
+				}
+				missing = len(data.addresses) - lastUsed
+			}
+			// check and generate change addresses
+			ca := data.changeAddresses
+			data.changeAddresses = make([]xpubAddress, len(data.addresses))
+			copy(data.changeAddresses, ca)
+			changeIndexes := []uint32{}
+			for i, ad := range data.addresses {
+				if ad.balance != nil {
+					if data.changeAddresses[i].addrDesc == nil {
+						changeIndexes = append(changeIndexes, uint32(i))
+					} else {
+						_, err := w.derivedAddressBalance(data, &ad)
+						if err != nil {
+							return nil, err
+						}
+					}
+				}
+			}
+			if len(changeIndexes) > 0 {
+				descriptors, err := w.chainParser.DeriveAddressDescriptors(xpub, 1, changeIndexes)
+				if err != nil {
+					return nil, err
+				}
+				for i, a := range descriptors {
+					ad := &data.changeAddresses[changeIndexes[i]]
+					ad.addrDesc = a
+					_, err := w.derivedAddressBalance(data, ad)
+					if err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	}
+	cachedXpubsMux.Lock()
+	cachedXpubs[xpub] = data
+	cachedXpubsMux.Unlock()
+	tokens := make([]Token, 0, 4)
+	for i, ad := range data.addresses {
+		if ad.balance != nil {
+			tokens = append(tokens, w.tokenFromXpubAddress(&ad, 0, i))
+		}
+		if data.changeAddresses[i].balance != nil {
+			tokens = append(tokens, w.tokenFromXpubAddress(&data.changeAddresses[i], 1, i))
+		}
+	}
+	var totalReceived big.Int
+	totalReceived.Add(&data.balanceSat, &data.sentSat)
+	addr := Address{
+		// Paging:                pg,
+		AddrStr:          xpub,
+		BalanceSat:       (*Amount)(&data.balanceSat),
+		TotalReceivedSat: (*Amount)(&totalReceived),
+		TotalSentSat:     (*Amount)(&data.sentSat),
+		Txs:              int(data.txs),
+		// UnconfirmedBalanceSat: (*Amount)(&uBalSat),
+		// UnconfirmedTxs:        len(txm),
+		// Transactions:          txs,
+		// Txids:                 txids,
+		Tokens: tokens,
+		// Erc20Contract:         erc20c,
+		// Nonce:                 nonce,
+	}
+	glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start))
+	return &addr, nil
+}

server/public.go

@@ -131,6 +131,7 @@ func (s *PublicServer) ConnectFullPublicInterface() {
 		// internal explorer handlers
 		serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx))
 		serveMux.HandleFunc(path+"address/", s.htmlTemplateHandler(s.explorerAddress))
+		serveMux.HandleFunc(path+"xpub/", s.htmlTemplateHandler(s.explorerXpub))
 		serveMux.HandleFunc(path+"search/", s.htmlTemplateHandler(s.explorerSearch))
 		serveMux.HandleFunc(path+"blocks", s.htmlTemplateHandler(s.explorerBlocks))
 		serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock))
@@ -165,6 +166,7 @@ func (s *PublicServer) ConnectFullPublicInterface() {
 	serveMux.HandleFunc(path+"api/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiDefault))
 	serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx, apiDefault))
 	serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress, apiDefault))
+	serveMux.HandleFunc(path+"api/xpub/", s.jsonHandler(s.apiXpub, apiDefault))
 	serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo, apiDefault))
 	serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock, apiDefault))
 	serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx, apiDefault))
@@ -174,6 +176,7 @@ func (s *PublicServer) ConnectFullPublicInterface() {
 	serveMux.HandleFunc(path+"api/v2/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV2))
 	serveMux.HandleFunc(path+"api/v2/tx/", s.jsonHandler(s.apiTx, apiV2))
 	serveMux.HandleFunc(path+"api/v2/address/", s.jsonHandler(s.apiAddress, apiV2))
+	serveMux.HandleFunc(path+"api/v2/xpub/", s.jsonHandler(s.apiXpub, apiV2))
 	serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV2))
 	serveMux.HandleFunc(path+"api/v2/block/", s.jsonHandler(s.apiBlock, apiV2))
 	serveMux.HandleFunc(path+"api/v2/sendtx/", s.jsonHandler(s.apiSendTx, apiV2))
@@ -372,6 +375,7 @@ const (
 	indexTpl
 	txTpl
 	addressTpl
+	xpubTpl
 	blocksTpl
 	blockTpl
 	sendTransactionTpl
@@ -465,6 +469,7 @@ func (s *PublicServer) parseTemplates() []*template.Template {
 		t[addressTpl] = createTemplate("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")
 		t[blockTpl] = createTemplate("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")
 	}
+	t[xpubTpl] = createTemplate("./static/templates/xpub.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")
 	return t
 }
 
@@ -577,6 +582,48 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) (
 	return addressTpl, data, nil
 }
 
+func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) {
+	var address *api.Address
+	var filter string
+	var fn = api.AddressFilterVoutOff
+	var err error
+	s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc()
+	if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 {
+		page, ec := strconv.Atoi(r.URL.Query().Get("page"))
+		if ec != nil {
+			page = 0
+		}
+		filter = r.URL.Query().Get("filter")
+		if len(filter) > 0 {
+			if filter == "inputs" {
+				fn = api.AddressFilterVoutInputs
+			} else if filter == "outputs" {
+				fn = api.AddressFilterVoutOutputs
+			} else {
+				fn, ec = strconv.Atoi(filter)
+				if ec != nil || fn < 0 {
+					filter = ""
+					fn = api.AddressFilterVoutOff
+				}
+			}
+		}
+		address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn})
+		if err != nil {
+			return errorTpl, nil, err
+		}
+	}
+	data := s.newTemplateData()
+	data.AddrStr = address.AddrStr
+	data.Address = address
+	data.Page = address.Page
+	data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages)
+	if filter != "" {
+		data.PageParams = template.URL("&filter=" + filter)
+		data.Address.Filter = filter
+	}
+	return xpubTpl, data, nil
+}
+
 func (s *PublicServer) explorerBlocks(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) {
 	var blocks *api.Blocks
 	var err error
@@ -638,6 +685,11 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t
 	var err error
 	s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc()
 	if len(q) > 0 {
+		address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff})
+		if err == nil {
+			http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302)
+			return noTpl, nil, nil
+		}
 		block, err = s.api.GetBlock(q, 0, 1)
 		if err == nil {
 			http.Redirect(w, r, joinURL("/block/", block.Hash), 302)
@@ -818,6 +870,23 @@ func (s *PublicServer) apiAddress(r *http.Request, apiVersion int) (interface{},
 	return address, err
 }
 
+func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, error) {
+	var address *api.Address
+	var err error
+	s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc()
+	if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 {
+		page, ec := strconv.Atoi(r.URL.Query().Get("page"))
+		if ec != nil {
+			page = 0
+		}
+		address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff})
+		if err == nil && apiVersion == apiV1 {
+			return s.api.AddressToV1(address), nil
+		}
+	}
+	return address, err
+}
+
 func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interface{}, error) {
 	var utxo []api.AddressUtxo
 	var err error

static/templates/base.html

@@ -42,7 +42,7 @@
                 </ul>
                 <span class="d-none ml-md-auto d-md-flex navbar-form navbar-left">
                     <form id="search" action="/search" method="get">
-                        <input name="q" type="text" class="form-control" placeholder="Search for block, transaction or address" focus="true">
+                        <input name="q" type="text" class="form-control" placeholder="Search for block, transaction, address or xpub" focus="true">
                     </form>
                 </span>
                 {{- end -}}

static/templates/xpub.html

@@ -0,0 +1,102 @@
+{{define "specific"}}{{$cs := .CoinShortcut}}{{$addr := .Address}}{{$data := .}}
+<h1>XPUB <small class="text-muted">{{formatAmount $addr.BalanceSat}} {{$cs}}</small>
+</h1>
+<div class="alert alert-data ellipsis">
+    <span class="data">{{$addr.AddrStr}}</span>
+</div>
+<h3>Confirmed</h3>
+<div class="data-div row">
+    <div class="col-md-10">
+        <table class="table data-table">
+            <tbody>
+                <tr>
+                    <td style="width: 25%;">Total Received</td>
+                    <td class="data">{{formatAmount $addr.TotalReceivedSat}} {{$cs}}</td>
+                </tr>
+                <tr>
+                    <td>Total Sent</td>
+                    <td class="data">{{formatAmount $addr.TotalSentSat}} {{$cs}}</td>
+                </tr>
+                <tr>
+                    <td>Final Balance</td>
+                    <td class="data">{{formatAmount $addr.BalanceSat}} {{$cs}}</td>
+                </tr>
+                <tr>
+                    <td>No. Transactions</td>
+                    <td class="data">{{$addr.Txs}}</td>
+                </tr>
+                {{- if $addr.Tokens -}}
+                <tr>
+                    <td>XPUB addresses</td>
+                    <td style="padding: 0;">
+                        <table class="table data-table">
+                            <tbody>
+                                <tr>
+                                    <th style="width: 50%;">Address</th>
+                                    <th>Balance</th>
+                                    <th style="width: 10%;">Txs</th>
+                                    <th style="width: 10%;">Path</th>
+                                </tr>
+                                {{- range $t := $addr.Tokens -}}
+                                <tr>
+                                    <td class="data ellipsis"><a href="/address/{{$t.Name}}">{{$t.Name}}</a></td>
+                                    <td class="data">{{formatAmount $t.BalanceSat}} {{$cs}}</td>
+                                    <td class="data">{{$t.Transfers}}</td>
+                                    <td class="data">{{$t.Contract}}</td>
+                                </tr>
+                                {{- end -}}
+                            </tbody>
+                        </table>
+                    </td>
+                </tr>
+                {{- end -}}
+            </tbody>
+        </table>
+    </div>
+    <div class="col-md-2">
+        <div id="qrcode" style="width: 160px; height: 160px;"></div>
+        <script type="text/javascript" src="/static/js/qrcode.min.js"></script>
+        <script type="text/javascript">
+            new QRCode(document.getElementById("qrcode"), { text: "{{$addr.AddrStr}}", width: 160, height: 160 });
+        </script>
+    </div>
+</div>
+{{- if $addr.UnconfirmedTxs -}}
+<h3>Unconfirmed</h3>
+<div class="data-div">
+    <table class="table data-table">
+        <tbody>
+            <tr>
+                <td style="width: 25%;">Unconfirmed Balance</td>
+                <td class="data">{{formatAmount $addr.UnconfirmedBalanceSat}} {{$cs}}</td>
+            </tr>
+            <tr>
+                <td>No. Transactions</td>
+                <td class="data">{{$addr.UnconfirmedTxs}}</td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+{{- end}}{{if or $addr.Transactions $addr.Filter -}}
+<div class="row h-container">
+    <h3 class="col-md-3">Transactions</h3>
+    <select class="col-md-2" style="background-color: #eaeaea;" onchange="self.location='?filter='+options[selectedIndex].value">
+        <option>All</option>
+        <option {{if eq $addr.Filter "inputs" -}} selected{{end}} value="inputs">Inputs</option>
+        <option {{if eq $addr.Filter "outputs" -}} selected{{end}} value="outputs">Outputs</option>
+        {{- if $addr.Tokens -}}
+        <option {{if eq $addr.Filter "0" -}} selected{{end}} value="0">Non-contract</option>
+        {{- range $t := $addr.Tokens -}}
+        <option {{if eq $addr.Filter $t.ContractIndex -}} selected{{end}} value="{{$t.ContractIndex}}">{{$t.Name}}</option>
+        {{- end -}}
+        {{- end -}}
+    </select>
+    <div class="col-md-7">
+        <nav>{{template "paging" $data}}</nav>
+    </div>
+</div>
+<div class="data-div">
+    {{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}}
+</div>
+<nav>{{template "paging" $data }}</nav>
+{{end}}{{end}}