Estimate full derivation path from xpub

Gopkg.lock

@@ -95,7 +95,7 @@
   branch = "master"
   name = "github.com/martinboehm/btcutil"
   packages = [".","base58","bech32","chaincfg","hdkeychain","txscript"]
-  revision = "520c2dbb6e0420531b6ae148845280378516e971"
+  revision = "63034958e64b209cb9294128309dbaed497cde7b"
 
 [[projects]]
   branch = "master"

api/xpub.go

@@ -45,6 +45,7 @@ type xpubAddress struct {
 
 type xpubData struct {
 	gap             int
+	basePath        string
 	dataHeight      uint32
 	dataHash        string
 	txs             uint32
@@ -208,7 +209,7 @@ func (w *Worker) xpubScanAddresses(xpub string, data *xpubData, addresses []xpub
 	return lastUsed, addresses, nil
 }
 
-func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token {
+func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeIndex int, index int) Token {
 	a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc)
 	var address string
 	if len(a) > 0 {
@@ -220,7 +221,7 @@ func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index in
 		Decimals:   w.chainParser.AmountDecimals(),
 		BalanceSat: (*Amount)(&ad.balance.BalanceSat),
 		Transfers:  int(ad.balance.Txs),
-		Contract:   fmt.Sprintf("%d/%d", changeIndex, index),
+		Contract:   fmt.Sprintf("%s/%d/%d", data.basePath, changeIndex, index),
 	}
 }
 
@@ -244,6 +245,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 	data, found := cachedXpubs[xpub]
 	cachedXpubsMux.Unlock()
 	var (
+		txc          xpubTxids
 		txm          []string
 		txs          []*Tx
 		txids        []string
@@ -265,6 +267,11 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 		fork := false
 		if !found || data.gap != gap {
 			data = &xpubData{gap: gap}
+			data.basePath, err = w.chainParser.DerivationBasePath(xpub)
+			if err != nil {
+				glog.Warning("DerivationBasePath error", err)
+				data.basePath = "unknown"
+			}
 		} else {
 			hash, err := w.db.GetBlockHash(data.dataHeight)
 			if err != nil {
@@ -276,9 +283,12 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 			}
 		}
 		processedHash = besthash
-		if data.dataHeight < bestheight {
+		if data.dataHeight < bestheight || fork {
 			data.dataHeight = bestheight
 			data.dataHash = besthash
+			data.balanceSat = *new(big.Int)
+			data.sentSat = *new(big.Int)
+			data.txs = 0
 			var lastUsedIndex int
 			lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, data, data.addresses, gap, 0, 0, fork)
 			if err != nil {
@@ -288,6 +298,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 			if err != nil {
 				return nil, err
 			}
+			glog.Info("Scanned ", len(data.addresses)+len(data.changeAddresses), " addresses in ", time.Since(start))
 		}
 		if option >= TxidHistory {
 			for i := range data.addresses {
@@ -307,7 +318,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 	cachedXpubsMux.Unlock()
 	// TODO mempool
 	if option >= TxidHistory {
-		txc := make(xpubTxids, 0, 32)
+		txc = make(xpubTxids, 0, 32)
 		var addTxids func(ad *xpubAddress)
 		if filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff {
 			addTxids = func(ad *xpubAddress) {
@@ -383,7 +394,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 		if ad.balance != nil {
 			totalTokens++
 			if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) {
-				t := w.tokenFromXpubAddress(ad, 0, i)
+				t := w.tokenFromXpubAddress(data, ad, 0, i)
 				tokens = append(tokens, t)
 				xpubAddresses[t.Name] = struct{}{}
 			} else {
@@ -399,7 +410,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 		if ad.balance != nil {
 			totalTokens++
 			if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) {
-				t := w.tokenFromXpubAddress(ad, 1, i)
+				t := w.tokenFromXpubAddress(data, ad, 1, i)
 				tokens = append(tokens, t)
 				xpubAddresses[t.Name] = struct{}{}
 			} else {
@@ -427,6 +438,6 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option
 		Tokens:        tokens,
 		XPubAddresses: xpubAddresses,
 	}
-	glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start))
+	glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs, loaded ", len(txc), " txids, finished in ", time.Since(start))
 	return &addr, nil
 }

bchain/baseparser.go

@@ -267,6 +267,11 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) {
 	return &tx, pt.Height, nil
 }
 
+// DerivationBasePath is unsupported
+func (p *BaseParser) DerivationBasePath(xpub string) (string, error) {
+	return "", errors.New("Not supported")
+}
+
 // DeriveAddressDescriptors is unsupported
 func (p *BaseParser) DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]AddressDescriptor, error) {
 	return nil, errors.New("Not supported")

bchain/coins/btc/bitcoinparser.go

@@ -6,6 +6,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"math/big"
+	"strconv"
 
 	vlq "github.com/bsm/go-vlq"
 	"github.com/juju/errors"
@@ -28,6 +29,7 @@ type BitcoinParser struct {
 	XPubMagic                   uint32
 	XPubMagicSegwitP2sh         uint32
 	XPubMagicSegwitNative       uint32
+	Slip44                      uint32
 }
 
 // NewBitcoinParser returns new BitcoinParser instance
@@ -41,6 +43,7 @@ func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser
 		XPubMagic:             c.XPubMagic,
 		XPubMagicSegwitP2sh:   c.XPubMagicSegwitP2sh,
 		XPubMagicSegwitNative: c.XPubMagicSegwitNative,
+		Slip44:                c.Slip44,
 	}
 	p.OutputScriptToAddressesFunc = p.outputScriptToAddresses
 	return p
@@ -349,3 +352,29 @@ func (p *BitcoinParser) DeriveAddressDescriptorsFromTo(xpub string, change uint3
 	}
 	return ad, nil
 }
+
+// DerivationBasePath returns base path of xpub
+func (p *BitcoinParser) DerivationBasePath(xpub string) (string, error) {
+	extKey, err := hdkeychain.NewKeyFromString(xpub)
+	if err != nil {
+		return "", err
+	}
+	var c, bip string
+	cn := extKey.ChildNum()
+	if cn >= 0x80000000 {
+		cn -= 0x80000000
+		c = "'"
+	}
+	c = strconv.Itoa(int(cn)) + c
+	if extKey.Depth() != 3 {
+		return "unknown/" + c, nil
+	}
+	if extKey.Version() == p.XPubMagicSegwitP2sh {
+		bip = "49"
+	} else if extKey.Version() == p.XPubMagicSegwitNative {
+		bip = "84"
+	} else {
+		bip = "44"
+	}
+	return "m/" + bip + "'/" + strconv.Itoa(int(p.Slip44)) + "'/" + c, nil
+}

bchain/coins/btc/bitcoinparser_test.go

@@ -590,3 +590,72 @@ func BenchmarkDeriveAddressDescriptorsFromToZpub(b *testing.B) {
 		btcMainParser.DeriveAddressDescriptorsFromTo("zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs", 1, 0, 100)
 	}
 }
+
+func TestBitcoinParser_DerivationBasePath(t *testing.T) {
+	btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518, Slip44: 0})
+	btcTestnetsParser := NewBitcoinParser(GetChainParams("test"), &Configuration{XPubMagic: 70617039, XPubMagicSegwitP2sh: 71979618, XPubMagicSegwitNative: 73342198, Slip44: 1})
+	zecMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, Slip44: 133})
+	type args struct {
+		xpub   string
+		parser *BitcoinParser
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "m/84'/0'/0'",
+			args: args{
+				xpub:   "zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs",
+				parser: btcMainParser,
+			},
+			want: "m/84'/0'/0'",
+		},
+		{
+			name: "m/49'/0'/55 - not hardened account",
+			args: args{
+				xpub:   "ypub6XKbB5DJRAbW4TRJLp4uXQXG3ob5BtByXsNZFBjq9qcbzrczjVXfCz5cEo1SFDexmeWRnbCMDaRgaW4m9d2nBaa8FvUQCu3n9G1UBR8WhbT",
+				parser: btcMainParser,
+			},
+			want: "m/49'/0'/55",
+		},
+		{
+			name: "m/49'/0' - incomplete path, without account",
+			args: args{
+				xpub:   "ypub6UzM8PUqxcSoqC9gumfoiFhE8Qt84HbGpCD4eVJfJAojXTVtBxeddvTWJGJhGoaVBNJLmEgMdLXHgaLVJa4xEvk2tcokkdZhFdkxMLUE9sB",
+				parser: btcMainParser,
+			},
+			want: "unknown/0'",
+		},
+		{
+			name: "m/49'/1'/0'",
+			args: args{
+				xpub:   "upub5DR1Mg5nykixzYjFXWW5GghAU7dDqoPVJ2jrqFbL8sJ7Hs7jn69MP7KBnnmxn88GeZtnH8PRKV9w5MMSFX8AdEAoXY8Qd8BJPoXtpMeHMxJ",
+				parser: btcTestnetsParser,
+			},
+			want: "m/49'/1'/0'",
+		},
+		{
+			name: "m/44'/133'/12'",
+			args: args{
+				xpub:   "xpub6CQdEahwhKRTLYpP6cyb7ZaGb3r4tVdyPX6dC1PfrNuByrCkWDgUkmpD28UdV9QccKgY1ZiAbGv1Fakcg2LxdFVSTNKHcjdRjqhjPK8Trkb",
+				parser: zecMainParser,
+			},
+			want: "m/44'/133'/12'",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.args.parser.DerivationBasePath(tt.args.xpub)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("BitcoinParser.DerivationBasePath() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("BitcoinParser.DerivationBasePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

bchain/coins/btc/bitcoinrpc.go

@@ -54,6 +54,7 @@ type Configuration struct {
 	XPubMagic                uint32 `json:"xpub_magic,omitempty"`
 	XPubMagicSegwitP2sh      uint32 `json:"xpub_magic_segwit_p2sh,omitempty"`
 	XPubMagicSegwitNative    uint32 `json:"xpub_magic_segwit_native,omitempty"`
+	Slip44                   uint32 `json:"slip44,omitempty"`
 }
 
 // NewBitcoinRPC returns new BitcoinRPC instance.

bchain/types.go

@@ -264,6 +264,7 @@ type BlockChainParser interface {
 	UnpackBlockHash(buf []byte) (string, error)
 	ParseBlock(b []byte) (*Block, error)
 	// xpub
+	DerivationBasePath(xpub string) (string, error)
 	DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]AddressDescriptor, error)
 	DeriveAddressDescriptorsFromTo(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error)
 	// EthereumType specific

build/templates/blockbook/blockchaincfg.json

@@ -19,6 +19,7 @@
 {{if .Blockbook.BlockChain.XPubMagic}}    "xpub_magic": {{.Blockbook.BlockChain.XPubMagic}},
 {{end}}{{if .Blockbook.BlockChain.XPubMagicSegwitP2sh}}    "xpub_magic_segwit_p2sh": {{.Blockbook.BlockChain.XPubMagicSegwitP2sh}},
 {{end}}{{if .Blockbook.BlockChain.XPubMagicSegwitNative}}    "xpub_magic_segwit_native": {{.Blockbook.BlockChain.XPubMagicSegwitNative}},
+{{end}}{{if .Blockbook.BlockChain.Slip44}}    "slip44": {{.Blockbook.BlockChain.Slip44}},
 {{end}}
     "mempool_workers": {{.Blockbook.BlockChain.MempoolWorkers}},
     "mempool_sub_workers": {{.Blockbook.BlockChain.MempoolSubWorkers}},

build/tools/templates.go

@@ -70,6 +70,7 @@ type Config struct {
 			XPubMagic             uint32 `json:"xpub_magic,omitempty"`
 			XPubMagicSegwitP2sh   uint32 `json:"xpub_magic_segwit_p2sh,omitempty"`
 			XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native,omitempty"`
+			Slip44                uint32 `json:"slip44,omitempty"`
 
 			AdditionalParams map[string]json.RawMessage `json:"additional_params"`
 		} `json:"block_chain"`

build/tools/trezor-common/sync-coins.go

@@ -27,6 +27,7 @@ type trezorCommonDef struct {
 	XPubMagic             uint32 `json:"xpub_magic"`
 	XPubMagicSegwitP2sh   uint32 `json:"xpub_magic_segwit_p2sh"`
 	XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native"`
+	Slip44                uint32 `json:"slip44,omitempty"`
 }
 
 func getTrezorCommonDef(coin string) (*trezorCommonDef, error) {
@@ -109,6 +110,9 @@ func main() {
 				if tcd.XPubMagicSegwitNative != 0 {
 					config.Blockbook.BlockChain.XPubMagicSegwitNative = tcd.XPubMagicSegwitNative
 				}
+				if tcd.Slip44 != 0 {
+					config.Blockbook.BlockChain.Slip44 = tcd.Slip44
+				}
 				err = writeConfig(coin, config)
 				if err == nil {
 					fmt.Printf("%v updated\n", coin)

configs/coins/bcash.json

@@ -55,6 +55,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
+      "slip44": 145,
       "additional_params": {}
     }
   },

configs/coins/bcash_testnet.json

@@ -55,6 +55,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 70617039,
+      "slip44": 1,
       "additional_params": {}
     }
   },

configs/coins/bgold.json

@@ -250,6 +250,7 @@
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
       "xpub_magic_segwit_p2sh": 77429938,
+      "slip44": 156,
       "additional_params": {}
     }
   },

configs/coins/bitcoin_testnet.json

@@ -58,6 +58,7 @@
       "xpub_magic": 70617039,
       "xpub_magic_segwit_p2sh": 71979618,
       "xpub_magic_segwit_native": 73342198,
+      "slip44": 1,
       "additional_params": {}
     }
   },

configs/coins/dash.json

@@ -57,6 +57,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 50221772,
+      "slip44": 5,
       "additional_params": {}
     }
   },

configs/coins/dash_testnet.json

@@ -57,6 +57,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 70617039,
+      "slip44": 1,
       "additional_params": {}
     }
   },

configs/coins/digibyte.json

@@ -57,6 +57,7 @@
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
       "xpub_magic_segwit_p2sh": 77429938,
+      "slip44": 20,
       "additional_params": {}
     }
   },

configs/coins/dogecoin.json

@@ -59,6 +59,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 49990397,
+      "slip44": 3,
       "additional_params": {}
     }
   },

configs/coins/fujicoin.json

@@ -54,6 +54,7 @@
       "xpub_magic": 76067358,
       "xpub_magic_segwit_p2sh": 77429938,
       "xpub_magic_segwit_native": 78792518,
+      "slip44": 75,
       "additional_params": {}
     }
   },

configs/coins/gamecredits.json

@@ -57,6 +57,7 @@
       "block_addresses_to_keep": 300,
       "xpub_magic": 27106558,
       "xpub_magic_segwit_p2sh": 28471030,
+      "slip44": 101,
       "additional_params": {}
     }
   },

configs/coins/groestlcoin.json

@@ -59,6 +59,7 @@
       "xpub_magic": 76067358,
       "xpub_magic_segwit_p2sh": 77429938,
       "xpub_magic_segwit_native": 78792518,
+      "slip44": 17,
       "additional_params": {}
     }
   },

configs/coins/groestlcoin_testnet.json

@@ -59,6 +59,7 @@
       "xpub_magic": 70617039,
       "xpub_magic_segwit_p2sh": 71979618,
       "xpub_magic_segwit_native": 73342198,
+      "slip44": 1,
       "additional_params": {}
     }
   },

configs/coins/koto.json

@@ -58,6 +58,7 @@
       "mempool_sub_workers": 8,
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
+      "slip44": 510,
       "additional_params": {}
     }
   },

configs/coins/litecoin.json

@@ -57,6 +57,7 @@
       "block_addresses_to_keep": 300,
       "xpub_magic": 27108450,
       "xpub_magic_segwit_p2sh": 28471030,
+      "slip44": 2,
       "additional_params": {}
     }
   },

configs/coins/litecoin_testnet.json

@@ -56,6 +56,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 70617039,
+      "slip44": 1,
       "additional_params": {}
     }
   },

configs/coins/monacoin.json

@@ -57,6 +57,7 @@
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
       "xpub_magic_segwit_p2sh": 77429938,
+      "slip44": 22,
       "additional_params": {}
     }
   },

configs/coins/myriad.json

@@ -56,6 +56,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
+      "slip44": 90,
       "additional_params": {}
     }
   },

configs/coins/namecoin.json

@@ -63,6 +63,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
+      "slip44": 7,
       "additional_params": {}
     }
   },

configs/coins/vertcoin.json

@@ -55,6 +55,7 @@
       "block_addresses_to_keep": 1000,
       "xpub_magic": 76067358,
       "xpub_magic_segwit_p2sh": 77429938,
+      "slip44": 28,
       "additional_params": {}
     }
   },

configs/coins/zcash.json

@@ -56,6 +56,7 @@
       "mempool_sub_workers": 8,
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
+      "slip44": 133,
       "additional_params": {}
     }
   },

configs/coins/zcash_testnet.json

@@ -56,6 +56,7 @@
       "mempool_sub_workers": 8,
       "block_addresses_to_keep": 300,
       "xpub_magic": 70617039,
+      "slip44": 1,
       "additional_params": {}
     }
   },

configs/coins/zcoin.json

@@ -67,6 +67,7 @@
       "mempool_sub_workers": 2,
       "block_addresses_to_keep": 300,
       "xpub_magic": 76067358,
+      "slip44": 136,
       "additional_params": {}
     }
   },

static/templates/xpub.html

@@ -39,14 +39,14 @@
                                     <th style="width: 50%;">Address</th>
                                     <th>Balance</th>
                                     <th style="width: 8%;">Txs</th>
-                                    <th style="width: 8%;">Path</th>
+                                    <th style="width: 18%;">Path</th>
                                 </tr>
                                 {{- range $t := $addr.Tokens -}}
                                 <tr>
                                     <td class="data ellipsis"><a href="/address/{{$t.Name}}">{{$t.Name}}</a></td>
                                     <td class="data">{{formatAmount $t.BalanceSat}} {{$cs}}</td>
                                     <td class="data">{{$t.Transfers}}</td>
-                                    <td class="data">{{$t.Contract}}</td>
+                                    <td>{{$t.Contract}}</td>
                                 </tr>
                                 {{- end -}}
                                 {{- if not $data.AllTokens -}}