From 42f6821cfacd792e34226efd167d9649ea821293 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 27 Jan 2019 00:29:44 +0100 Subject: [PATCH 01/26] Update gitignore --- .gitignore | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 4f23b5df..8d866cba 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,10 @@ notes.txt debug* .vscode docker/blockbook -build -!build/templates -!build/docker +build/pkg-defs +build/blockbook +build/ldb +build/sst_dump +build/*.deb .bin-image .deb-image \ No newline at end of file From dafe19cf2932b6f3c6a18e421a409a28e74434bc Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 28 Jan 2019 13:34:43 +0100 Subject: [PATCH 02/26] Synchronize xpub magic numbers from trezor-common --- build/templates/blockbook/blockchaincfg.json | 7 +- build/tools/templates.go | 47 +- build/tools/trezor-common/sync-coins.go | 122 ++++++ configs/coins/bcash.json | 13 +- configs/coins/bcash_testnet.json | 13 +- configs/coins/bgold.json | 400 +++++++++--------- configs/coins/bitcoin.json | 17 +- configs/coins/bitcoin_testnet.json | 17 +- configs/coins/dash.json | 15 +- configs/coins/dash_testnet.json | 15 +- configs/coins/digibyte.json | 16 +- configs/coins/dogecoin.json | 21 +- configs/coins/fujicoin.json | 19 +- configs/coins/gamecredits.json | 16 +- configs/coins/groestlcoin.json | 19 +- configs/coins/groestlcoin_testnet.json | 19 +- configs/coins/koto.json | 19 +- configs/coins/litecoin.json | 16 +- configs/coins/litecoin_testnet.json | 15 +- configs/coins/monacoin.json | 16 +- configs/coins/myriad.json | 15 +- configs/coins/namecoin.json | 29 +- configs/coins/vertcoin.json | 14 +- configs/coins/zcash.json | 17 +- configs/coins/zcash_testnet.json | 17 +- configs/coins/zcoin.json | 13 +- .../check-and-generate-port-registry.go | 2 +- docs/ports.md | 3 +- 28 files changed, 559 insertions(+), 393 deletions(-) create mode 100644 build/tools/trezor-common/sync-coins.go diff --git a/build/templates/blockbook/blockchaincfg.json b/build/templates/blockbook/blockchaincfg.json index 159eba57..aea2ad96 100644 --- a/build/templates/blockbook/blockchaincfg.json +++ b/build/templates/blockbook/blockchaincfg.json @@ -3,9 +3,8 @@ {{- if .Blockbook.BlockChain.AdditionalParams}} {{- range $name, $value := .Blockbook.BlockChain.AdditionalParams}} "{{$name}}": {{jsonToString $value}}, -{{- end}} +{{- end -}} {{end}} - "coin_name": "{{.Coin.Name}}", "coin_shortcut": "{{.Coin.Shortcut}}", "coin_label": "{{.Coin.Label}}", @@ -17,6 +16,10 @@ "message_queue_binding": "{{template "IPC.MessageQueueBindingTemplate" .}}", "subversion": "{{.Blockbook.BlockChain.Subversion}}", "address_format": "{{.Blockbook.BlockChain.AddressFormat}}", +{{if .Blockbook.BlockChain.XPubMagic}} "xpub_magic": {{.Blockbook.BlockChain.XPubMagic}}, +{{end}}{{if .Blockbook.BlockChain.XPubMagicSegwitP2sh}} "xpub_magic_segwit_p2sh": {{.Blockbook.BlockChain.XPubMagicSegwitP2sh}}, +{{end}}{{if .Blockbook.BlockChain.XPubMagicSegwitNative}} "xpub_magic_segwit_native": {{.Blockbook.BlockChain.XPubMagicSegwitNative}}, +{{end}} "mempool_workers": {{.Blockbook.BlockChain.MempoolWorkers}}, "mempool_sub_workers": {{.Blockbook.BlockChain.MempoolSubWorkers}}, "block_addresses_to_keep": {{.Blockbook.BlockChain.BlockAddressesToKeep}} diff --git a/build/tools/templates.go b/build/tools/templates.go index ebd9d93a..5194cd98 100644 --- a/build/tools/templates.go +++ b/build/tools/templates.go @@ -13,18 +13,6 @@ import ( ) type Config struct { - Meta struct { - BuildDatetime string // generated field - PackageMaintainer string `json:"package_maintainer"` - PackageMaintainerEmail string `json:"package_maintainer_email"` - } - Env struct { - Version string `json:"version"` - BackendInstallPath string `json:"backend_install_path"` - BackendDataPath string `json:"backend_data_path"` - BlockbookInstallPath string `json:"blockbook_install_path"` - BlockbookDataPath string `json:"blockbook_data_path"` - } `json:"env"` Coin struct { Name string `json:"name"` Shortcut string `json:"shortcut"` @@ -63,7 +51,7 @@ type Config struct { Mainnet bool `json:"mainnet"` ServerConfigFile string `json:"server_config_file"` ClientConfigFile string `json:"client_config_file"` - AdditionalParams interface{} `json:"additional_params"` + AdditionalParams interface{} `json:"additional_params,omitempty"` } `json:"backend"` Blockbook struct { PackageName string `json:"package_name"` @@ -73,16 +61,31 @@ type Config struct { ExplorerURL string `json:"explorer_url"` AdditionalParams string `json:"additional_params"` BlockChain struct { - Parse bool `json:"parse"` - Subversion string `json:"subversion"` - AddressFormat string `json:"address_format"` - MempoolWorkers int `json:"mempool_workers"` - MempoolSubWorkers int `json:"mempool_sub_workers"` - BlockAddressesToKeep int `json:"block_addresses_to_keep"` - AdditionalParams map[string]json.RawMessage `json:"additional_params"` + Parse bool `json:"parse,omitempty"` + Subversion string `json:"subversion,omitempty"` + AddressFormat string `json:"address_format,omitempty"` + MempoolWorkers int `json:"mempool_workers"` + MempoolSubWorkers int `json:"mempool_sub_workers"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` + XPubMagic uint32 `json:"xpub_magic,omitempty"` + XPubMagicSegwitP2sh uint32 `json:"xpub_magic_segwit_p2sh,omitempty"` + XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native,omitempty"` + + AdditionalParams map[string]json.RawMessage `json:"additional_params"` } `json:"block_chain"` } `json:"blockbook"` - IntegrationTests map[string][]string `json:"integration_tests"` + Meta struct { + BuildDatetime string `json:"-"` // generated field + PackageMaintainer string `json:"package_maintainer"` + PackageMaintainerEmail string `json:"package_maintainer_email"` + } `json:"meta"` + Env struct { + Version string `json:"version"` + BackendInstallPath string `json:"backend_install_path"` + BackendDataPath string `json:"backend_data_path"` + BlockbookInstallPath string `json:"blockbook_install_path"` + BlockbookDataPath string `json:"blockbook_data_path"` + } `json:"-"` } func jsonToString(msg json.RawMessage) (string, error) { @@ -266,7 +269,7 @@ func makeOutputDir(path string) error { } func writeTemplate(path string, info os.FileInfo, templ *template.Template, config *Config) error { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, info.Mode()) + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode()) if err != nil { return err } diff --git a/build/tools/trezor-common/sync-coins.go b/build/tools/trezor-common/sync-coins.go new file mode 100644 index 00000000..a8aa4be0 --- /dev/null +++ b/build/tools/trezor-common/sync-coins.go @@ -0,0 +1,122 @@ +//usr/bin/go run $0 $@ ; exit +package main + +import ( + build "blockbook/build/tools" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" +) + +const ( + configsDir = "configs" + trezorCommonDefsURL = "https://raw.githubusercontent.com/trezor/trezor-common/master/defs/bitcoin/" +) + +type trezorCommonDef struct { + Name string `json:"coin_name"` + Shortcut string `json:"coin_shortcut"` + Label string `json:"coin_label"` + XPubMagic uint32 `json:"xpub_magic"` + XPubMagicSegwitP2sh uint32 `json:"xpub_magic_segwit_p2sh"` + XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native"` +} + +func getTrezorCommonDef(coin string) (*trezorCommonDef, error) { + req, err := http.NewRequest("GET", trezorCommonDefsURL+coin+".json", nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errors.New("Github request status code " + strconv.Itoa(resp.StatusCode)) + } + bb, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var tcd trezorCommonDef + json.Unmarshal(bb, &tcd) + return &tcd, nil +} + +func writeConfig(coin string, config *build.Config) error { + path := filepath.Join(configsDir, "coins", coin+".json") + out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer out.Close() + buf, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + n, err := out.Write(buf) + if err != nil { + return err + } + if n < len(buf) { + return io.ErrShortWrite + } + return nil +} + +func main() { + var coins []string + if len(os.Args) < 2 { + filepath.Walk(filepath.Join(configsDir, "coins"), func(path string, info os.FileInfo, err error) error { + n := strings.TrimSuffix(info.Name(), ".json") + if n != info.Name() { + coins = append(coins, n) + } + return nil + }) + } else { + coins = append(coins, os.Args[1]) + } + for _, coin := range coins { + config, err := build.LoadConfig(configsDir, coin) + if err == nil { + var tcd *trezorCommonDef + tcd, err = getTrezorCommonDef(coin) + if err == nil { + if tcd.Name != "" { + config.Coin.Name = tcd.Name + } + if tcd.Shortcut != "" { + config.Coin.Shortcut = tcd.Shortcut + } + if tcd.Label != "" { + config.Coin.Label = tcd.Label + } + if tcd.XPubMagic != 0 { + config.Blockbook.BlockChain.XPubMagic = tcd.XPubMagic + } + if tcd.XPubMagicSegwitP2sh != 0 { + config.Blockbook.BlockChain.XPubMagicSegwitP2sh = tcd.XPubMagicSegwitP2sh + } + if tcd.XPubMagicSegwitNative != 0 { + config.Blockbook.BlockChain.XPubMagicSegwitNative = tcd.XPubMagicSegwitNative + } + err = writeConfig(coin, config) + if err == nil { + fmt.Printf("%v updated\n", coin) + } + } + } + if err != nil { + fmt.Printf("%v update error %v\n", coin, err) + } + } +} diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 76ed30a5..365e925f 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Bcash", - "shortcut": "BCH", - "label": "Bitcoin Cash", - "alias": "bcash" + "name": "Bcash", + "shortcut": "BCH", + "label": "Bitcoin Cash", + "alias": "bcash" }, "ports": { "backend_rpc": 8031, @@ -28,7 +28,7 @@ "verification_source": "4de8e8c4679868befb3865ac23187dcb75824ccb4e29847efdb451df274bd65b", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/bitcoin-qt" + "bin/bitcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bitcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -54,6 +54,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, "additional_params": {} } }, @@ -61,4 +62,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index ec55f587..f8b65e7a 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Bcash Testnet", - "shortcut": "TBCH", - "label": "Bitcoin Cash Testnet", - "alias": "bcash_testnet" + "name": "Bcash Testnet", + "shortcut": "TBCH", + "label": "Bitcoin Cash Testnet", + "alias": "bcash_testnet" }, "ports": { "backend_rpc": 18031, @@ -28,7 +28,7 @@ "verification_source": "4de8e8c4679868befb3865ac23187dcb75824ccb4e29847efdb451df274bd65b", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/bitcoin-qt" + "bin/bitcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bitcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet3/*.log", @@ -54,6 +54,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 70617039, "additional_params": {} } }, @@ -61,4 +62,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/bgold.json b/configs/coins/bgold.json index 01d225e4..063868b4 100644 --- a/configs/coins/bgold.json +++ b/configs/coins/bgold.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Bgold", - "shortcut": "BTG", - "label": "Bitcoin Gold", - "alias": "bgold" + "name": "Bgold", + "shortcut": "BTG", + "label": "Bitcoin Gold", + "alias": "bgold" }, "ports": { "backend_rpc": 8035, @@ -28,7 +28,7 @@ "verification_source": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.2/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/bitcoin-qt" + "bin/bitcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bgoldd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,199 +40,199 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "mempoolexpiry": 72, - "timeout": 768, - "maxconnections": 250, - "addnode": [ - "188.126.0.134", - "45.56.84.44", - "109.201.133.93:8338", - "178.63.11.246:8338", - "188.120.223.153:8338", - "79.137.64.158:8338", - "78.193.221.106:8338", - "139.59.151.13:8338", - "76.16.12.81:8338", - "172.104.157.62:8338", - "43.207.67.209:8338", - "178.63.11.246:8338", - "79.137.64.158:8338", - "78.193.221.106:8338", - "139.59.151.13:8338", - "172.104.157.62:8338", - "178.158.247.119:8338", - "109.201.133.93:8338", - "178.63.11.246:8338", - "139.59.151.13:8338", - "172.104.157.62:8338", - "188.120.223.153:8338", - "178.158.247.119:8338", - "78.193.221.106:8338", - "79.137.64.158:8338", - "76.16.12.81:8338", - "176.12.32.153:8338", - "178.158.247.122:8338", - "81.37.147.185:8338", - "176.12.32.153:8338", - "79.137.64.158:8338", - "178.158.247.122:8338", - "66.70.247.151:8338", - "89.18.27.165:8338", - "178.63.11.246:8338", - "91.222.17.86:8338", - "37.59.50.143:8338", - "91.50.219.221:8338", - "154.16.63.17:8338", - "213.136.76.42:8338", - "176.99.4.140:8338", - "176.9.48.36:8338", - "78.193.221.106:8338", - "34.236.228.99:8338", - "213.154.230.107:8338", - "111.231.66.252:8338", - "188.120.223.153:8338", - "219.89.122.82:8338", - "109.192.23.101:8338", - "98.114.91.222:8338", - "217.66.156.41:8338", - "172.104.157.62:8338", - "114.44.222.73:8338", - "91.224.140.216:8338", - "149.154.71.96:8338", - "107.181.183.242:8338", - "36.78.96.92:8338", - "46.22.7.74:8338", - "89.110.53.186:8338", - "73.243.220.85:8338", - "109.86.137.8:8338", - "77.78.12.89:8338", - "87.92.116.26:8338", - "93.78.122.48:8338", - "35.195.83.0:8338", - "46.147.75.220:8338", - "212.47.236.104:8338", - "95.220.100.230:8338", - "178.70.142.247:8338", - "45.76.136.149:8338", - "94.155.74.206:8338", - "178.70.142.247:8338", - "128.199.228.97:8338", - "77.171.144.207:8338", - "159.89.192.119:8338", - "136.63.238.170:8338", - "31.27.193.105:8338", - "176.107.192.240:8338", - "94.140.241.96:8338", - "66.108.15.5:8338", - "81.177.127.204:8338", - "88.18.69.174:8338", - "178.70.130.94:8338", - "78.98.162.140:8338", - "95.133.156.224:8338", - "46.188.16.96:8338", - "94.247.16.21:8338", - "eunode.pool.gold:8338", - "asianode.pool.gold:8338", - "45.56.84.44:8338", - "176.9.48.36:8338", - "93.57.253.121:8338", - "172.104.157.62:8338", - "176.12.32.153:8338", - "pool.serverpower.net:8338", - "213.154.229.126:8338", - "213.154.230.106:8338", - "213.154.230.107:8338", - "213.154.229.50:8338", - "145.239.0.50:8338", - "107.181.183.242:8338", - "109.201.133.93:8338", - "120.41.190.109:8338", - "120.41.191.224:8338", - "138.68.249.79:8338", - "13.95.223.202:8338", - "145.239.0.50:8338", - "149.56.95.26:8338", - "158.69.103.228:8338", - "159.89.192.119:8338", - "164.132.207.143:8338", - "171.100.141.106:8338", - "172.104.157.62:8338", - "173.176.95.92:8338", - "176.12.32.153:8338", - "178.239.54.250:8338", - "178.63.11.246:8338", - "185.139.2.140:8338", - "188.120.223.153:8338", - "190.46.2.92:8338", - "192.99.194.113:8338", - "199.229.248.218:8338", - "213.154.229.126:8338", - "213.154.229.50:8338", - "213.154.230.106:8338", - "213.154.230.107:8338", - "217.182.199.21", - "35.189.127.200:8338", - "35.195.83.0:8338", - "35.197.197.166:8338", - "35.200.168.155:8338", - "35.203.167.11:8338", - "37.59.50.143:8338", - "45.27.161.195:8338", - "45.32.234.160:8338", - "45.56.84.44:8338", - "46.188.16.96:8338", - "46.251.19.171:8338", - "5.157.119.109:8338", - "52.28.162.48:8338", - "54.153.140.202:8338", - "54.68.81.2:83388338", - "62.195.190.190:8338", - "62.216.5.136:8338", - "65.110.125.175:8338", - "67.68.226.130:8338", - "73.243.220.85:8338", - "77.78.12.89:8338", - "78.193.221.106:8338", - "78.98.162.140:8338", - "79.137.64.158:8338", - "84.144.177.238:8338", - "87.92.116.26:8338", - "89.115.139.117:8338", - "89.18.27.165:8338", - "91.50.219.221:8338", - "93.88.74.26", - "93.88.74.26:8338", - "94.155.74.206:8338", - "95.154.201.132:8338", - "98.29.248.131:8338", - "u2.my.to:8338", - "[2001:470:b:ce:dc70:83ff:fe7a:1e74]:8338", - "2001:7b8:61d:1:250:56ff:fe90:c89f:8338", - "2001:7b8:63a:1002:213:154:230:106:8338", - "2001:7b8:63a:1002:213:154:230:107:8338", - "45.56.84.44", - "109.201.133.93:8338", - "120.41.191.224:30607", - "138.68.249.79:50992", - "138.68.249.79:51314", - "172.104.157.62", - "178.63.11.246:8338", - "185.139.2.140:8338", - "199.229.248.218:28830", - "35.189.127.200:41220", - "35.189.127.200:48244", - "35.195.83.0:35172", - "35.195.83.0:35576", - "35.195.83.0:35798", - "35.197.197.166:32794", - "35.197.197.166:33112", - "35.197.197.166:33332", - "35.203.167.11:52158", - "37.59.50.143:35254", - "45.27.161.195:33852", - "45.27.161.195:36738", - "45.27.161.195:58628" - ] + "addnode": [ + "188.126.0.134", + "45.56.84.44", + "109.201.133.93:8338", + "178.63.11.246:8338", + "188.120.223.153:8338", + "79.137.64.158:8338", + "78.193.221.106:8338", + "139.59.151.13:8338", + "76.16.12.81:8338", + "172.104.157.62:8338", + "43.207.67.209:8338", + "178.63.11.246:8338", + "79.137.64.158:8338", + "78.193.221.106:8338", + "139.59.151.13:8338", + "172.104.157.62:8338", + "178.158.247.119:8338", + "109.201.133.93:8338", + "178.63.11.246:8338", + "139.59.151.13:8338", + "172.104.157.62:8338", + "188.120.223.153:8338", + "178.158.247.119:8338", + "78.193.221.106:8338", + "79.137.64.158:8338", + "76.16.12.81:8338", + "176.12.32.153:8338", + "178.158.247.122:8338", + "81.37.147.185:8338", + "176.12.32.153:8338", + "79.137.64.158:8338", + "178.158.247.122:8338", + "66.70.247.151:8338", + "89.18.27.165:8338", + "178.63.11.246:8338", + "91.222.17.86:8338", + "37.59.50.143:8338", + "91.50.219.221:8338", + "154.16.63.17:8338", + "213.136.76.42:8338", + "176.99.4.140:8338", + "176.9.48.36:8338", + "78.193.221.106:8338", + "34.236.228.99:8338", + "213.154.230.107:8338", + "111.231.66.252:8338", + "188.120.223.153:8338", + "219.89.122.82:8338", + "109.192.23.101:8338", + "98.114.91.222:8338", + "217.66.156.41:8338", + "172.104.157.62:8338", + "114.44.222.73:8338", + "91.224.140.216:8338", + "149.154.71.96:8338", + "107.181.183.242:8338", + "36.78.96.92:8338", + "46.22.7.74:8338", + "89.110.53.186:8338", + "73.243.220.85:8338", + "109.86.137.8:8338", + "77.78.12.89:8338", + "87.92.116.26:8338", + "93.78.122.48:8338", + "35.195.83.0:8338", + "46.147.75.220:8338", + "212.47.236.104:8338", + "95.220.100.230:8338", + "178.70.142.247:8338", + "45.76.136.149:8338", + "94.155.74.206:8338", + "178.70.142.247:8338", + "128.199.228.97:8338", + "77.171.144.207:8338", + "159.89.192.119:8338", + "136.63.238.170:8338", + "31.27.193.105:8338", + "176.107.192.240:8338", + "94.140.241.96:8338", + "66.108.15.5:8338", + "81.177.127.204:8338", + "88.18.69.174:8338", + "178.70.130.94:8338", + "78.98.162.140:8338", + "95.133.156.224:8338", + "46.188.16.96:8338", + "94.247.16.21:8338", + "eunode.pool.gold:8338", + "asianode.pool.gold:8338", + "45.56.84.44:8338", + "176.9.48.36:8338", + "93.57.253.121:8338", + "172.104.157.62:8338", + "176.12.32.153:8338", + "pool.serverpower.net:8338", + "213.154.229.126:8338", + "213.154.230.106:8338", + "213.154.230.107:8338", + "213.154.229.50:8338", + "145.239.0.50:8338", + "107.181.183.242:8338", + "109.201.133.93:8338", + "120.41.190.109:8338", + "120.41.191.224:8338", + "138.68.249.79:8338", + "13.95.223.202:8338", + "145.239.0.50:8338", + "149.56.95.26:8338", + "158.69.103.228:8338", + "159.89.192.119:8338", + "164.132.207.143:8338", + "171.100.141.106:8338", + "172.104.157.62:8338", + "173.176.95.92:8338", + "176.12.32.153:8338", + "178.239.54.250:8338", + "178.63.11.246:8338", + "185.139.2.140:8338", + "188.120.223.153:8338", + "190.46.2.92:8338", + "192.99.194.113:8338", + "199.229.248.218:8338", + "213.154.229.126:8338", + "213.154.229.50:8338", + "213.154.230.106:8338", + "213.154.230.107:8338", + "217.182.199.21", + "35.189.127.200:8338", + "35.195.83.0:8338", + "35.197.197.166:8338", + "35.200.168.155:8338", + "35.203.167.11:8338", + "37.59.50.143:8338", + "45.27.161.195:8338", + "45.32.234.160:8338", + "45.56.84.44:8338", + "46.188.16.96:8338", + "46.251.19.171:8338", + "5.157.119.109:8338", + "52.28.162.48:8338", + "54.153.140.202:8338", + "54.68.81.2:83388338", + "62.195.190.190:8338", + "62.216.5.136:8338", + "65.110.125.175:8338", + "67.68.226.130:8338", + "73.243.220.85:8338", + "77.78.12.89:8338", + "78.193.221.106:8338", + "78.98.162.140:8338", + "79.137.64.158:8338", + "84.144.177.238:8338", + "87.92.116.26:8338", + "89.115.139.117:8338", + "89.18.27.165:8338", + "91.50.219.221:8338", + "93.88.74.26", + "93.88.74.26:8338", + "94.155.74.206:8338", + "95.154.201.132:8338", + "98.29.248.131:8338", + "u2.my.to:8338", + "[2001:470:b:ce:dc70:83ff:fe7a:1e74]:8338", + "2001:7b8:61d:1:250:56ff:fe90:c89f:8338", + "2001:7b8:63a:1002:213:154:230:106:8338", + "2001:7b8:63a:1002:213:154:230:107:8338", + "45.56.84.44", + "109.201.133.93:8338", + "120.41.191.224:30607", + "138.68.249.79:50992", + "138.68.249.79:51314", + "172.104.157.62", + "178.63.11.246:8338", + "185.139.2.140:8338", + "199.229.248.218:28830", + "35.189.127.200:41220", + "35.189.127.200:48244", + "35.195.83.0:35172", + "35.195.83.0:35576", + "35.195.83.0:35798", + "35.197.197.166:32794", + "35.197.197.166:33112", + "35.197.197.166:33332", + "35.203.167.11:52158", + "37.59.50.143:35254", + "45.27.161.195:33852", + "45.27.161.195:36738", + "45.27.161.195:58628" + ], + "maxconnections": 250, + "mempoolexpiry": 72, + "timeout": 768 } }, "blockbook": { @@ -248,6 +248,8 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, "additional_params": {} } }, @@ -255,4 +257,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index 1532fd53..c9dbf261 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Bitcoin", - "shortcut": "BTC", - "label": "Bitcoin", - "alias": "bitcoin" + "name": "Bitcoin", + "shortcut": "BTC", + "label": "Bitcoin", + "alias": "bitcoin" }, "ports": { "backend_rpc": 8030, @@ -28,7 +28,7 @@ "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.17.1/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/bitcoin-qt" + "bin/bitcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bitcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin.conf", "client_config_file": "bitcoin_client.conf", "additional_params": { - "deprecatedrpc": "estimatefee" + "deprecatedrpc": "estimatefee" } }, "blockbook": { @@ -55,6 +55,9 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "additional_params": {} } }, @@ -62,4 +65,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index ea424cab..e1136f46 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Testnet", - "shortcut": "TEST", - "label": "Bitcoin Testnet", - "alias": "bitcoin_testnet" + "name": "Testnet", + "shortcut": "TEST", + "label": "Bitcoin Testnet", + "alias": "bitcoin_testnet" }, "ports": { "backend_rpc": 18030, @@ -28,7 +28,7 @@ "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.17.1/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/bitcoin-qt" + "bin/bitcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bitcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet3/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin.conf", "client_config_file": "bitcoin_client.conf", "additional_params": { - "deprecatedrpc": "estimatefee" + "deprecatedrpc": "estimatefee" } }, "blockbook": { @@ -55,6 +55,9 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 70617039, + "xpub_magic_segwit_p2sh": 71979618, + "xpub_magic_segwit_native": 73342198, "additional_params": {} } }, @@ -62,4 +65,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/dash.json b/configs/coins/dash.json index ea160e98..7ddd1c58 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Dash", - "shortcut": "DASH", - "label": "Dash", - "alias": "dash" + "name": "Dash", + "shortcut": "DASH", + "label": "Dash", + "alias": "dash" }, "ports": { "backend_rpc": 8033, @@ -28,7 +28,7 @@ "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.0.0/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/dash-qt" + "bin/dash-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/dashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "mempoolexpiry": 72 + "mempoolexpiry": 72 } }, "blockbook": { @@ -56,6 +56,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 50221772, "additional_params": {} } }, @@ -63,4 +64,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index 98ae8fc6..94b7cfae 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Dash Testnet", - "shortcut": "tDASH", - "label": "Dash Testnet", - "alias": "dash_testnet" + "name": "Dash Testnet", + "shortcut": "tDASH", + "label": "Dash Testnet", + "alias": "dash_testnet" }, "ports": { "backend_rpc": 18033, @@ -28,7 +28,7 @@ "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.0.0/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/dash-qt" + "bin/dash-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/dashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet3/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "mempoolexpiry": 72 + "mempoolexpiry": 72 } }, "blockbook": { @@ -56,6 +56,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 70617039, "additional_params": {} } }, @@ -63,4 +64,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/digibyte.json b/configs/coins/digibyte.json index 3608a65d..08b4a47d 100644 --- a/configs/coins/digibyte.json +++ b/configs/coins/digibyte.json @@ -1,9 +1,9 @@ { "coin": { - "name": "DigiByte", - "shortcut": "DGB", - "label": "DigiByte", - "alias": "digibyte" + "name": "DigiByte", + "shortcut": "DGB", + "label": "DigiByte", + "alias": "digibyte" }, "ports": { "backend_rpc": 8042, @@ -28,7 +28,7 @@ "verification_source": "dd6bed0228087fbb51f08be55cbc08a0e3251acfe1be3249b634447837ecd857", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/digibyte-qt" + "bin/digibyte-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/digibyted -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -55,6 +55,8 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, "additional_params": {} } }, @@ -62,4 +64,4 @@ "package_maintainer": "Martin Boehm", "package_maintainer_email": "martin.bohm@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/dogecoin.json b/configs/coins/dogecoin.json index d3c44e00..1fcdc8df 100644 --- a/configs/coins/dogecoin.json +++ b/configs/coins/dogecoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Dogecoin", - "shortcut": "DOGE", - "label": "Dogecoin", - "alias": "dogecoin" + "name": "Dogecoin", + "shortcut": "DOGE", + "label": "Dogecoin", + "alias": "dogecoin" }, "ports": { "backend_rpc": 8038, @@ -28,7 +28,7 @@ "verification_source": "09871d8ff2ab5e0f05df2bdf5eba64c178229d030dd7c8473b08e6ed45d3327f", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/dogecoin-qt" + "bin/dogecoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/dogecoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,10 +40,10 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1", - "rpcthreads": 16, - "upnp": 0, - "discover": 0 + "discover": 0, + "rpcthreads": 16, + "upnp": 0, + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -58,6 +58,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 49990397, "additional_params": {} } }, @@ -65,4 +66,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/fujicoin.json b/configs/coins/fujicoin.json index b0f02543..0431b17d 100644 --- a/configs/coins/fujicoin.json +++ b/configs/coins/fujicoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Fujicoin", - "shortcut": "FJC", - "label": "Fujicoin", - "alias": "fujicoin" + "name": "Fujicoin", + "shortcut": "FJC", + "label": "Fujicoin", + "alias": "fujicoin" }, "ports": { "backend_rpc": 8048, @@ -27,8 +27,7 @@ "verification_type": "gpg-sha256", "verification_source": "https://www.fujicoin.org/fujicoin/3.0/SHA256SUMS.asc", "extract_command": "tar -C backend -xf", - "exclude_files": [ - ], + "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/fujicoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", "postinst_script_template": "", @@ -38,8 +37,7 @@ "mainnet": true, "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", - "additional_params": { - } + "additional_params": {} }, "blockbook": { "package_name": "blockbook-fujicoin", @@ -53,6 +51,9 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "additional_params": {} } }, @@ -60,4 +61,4 @@ "package_maintainer": "Motty", "package_maintainer_email": "fujicoin@gmail.com" } -} +} \ No newline at end of file diff --git a/configs/coins/gamecredits.json b/configs/coins/gamecredits.json index bfdb5744..7071bd82 100644 --- a/configs/coins/gamecredits.json +++ b/configs/coins/gamecredits.json @@ -1,9 +1,9 @@ { "coin": { - "name": "GameCredits", - "shortcut": "GAME", - "label": "GameCredits", - "alias": "gamecredits" + "name": "GameCredits", + "shortcut": "GAME", + "label": "GameCredits", + "alias": "gamecredits" }, "ports": { "backend_rpc": 8044, @@ -28,7 +28,7 @@ "verification_source": "38531ea877dfc1cedd3125bb79216a587f0974f20bee6243efcde61d05e07e5c", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/gamecredits-qt" + "bin/gamecredits-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/gamecreditsd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -55,6 +55,8 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 27106558, + "xpub_magic_segwit_p2sh": 28471030, "additional_params": {} } }, @@ -62,4 +64,4 @@ "package_maintainer": "Samad Sajanlal", "package_maintainer_email": "samad@gamecredits.org" } -} +} \ No newline at end of file diff --git a/configs/coins/groestlcoin.json b/configs/coins/groestlcoin.json index 760799da..6ac5901a 100644 --- a/configs/coins/groestlcoin.json +++ b/configs/coins/groestlcoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Groestlcoin", - "shortcut": "GRS", - "label": "Groestlcoin", - "alias": "groestlcoin" + "name": "Groestlcoin", + "shortcut": "GRS", + "label": "Groestlcoin", + "alias": "groestlcoin" }, "ports": { "backend_rpc": 8045, @@ -28,7 +28,7 @@ "verification_source": "f15bd5e38b25a103821f1563cd0e1b2cf7146ec9f9835493a30bd57313d3b86f", "extract_command": "mkdir -p backend/bin; tar -C backend/bin -xf", "exclude_files": [ - "bin/groestlcoin-qt" + "bin/groestlcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/groestlcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,8 +40,8 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "deprecatedrpc": "estimatefee", - "whitelist": "127.0.0.1" + "deprecatedrpc": "estimatefee", + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -56,6 +56,9 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "additional_params": {} } }, @@ -63,4 +66,4 @@ "package_maintainer": "Groestlcoin team", "package_maintainer_email": "support@groestlcoin.org" } -} +} \ No newline at end of file diff --git a/configs/coins/groestlcoin_testnet.json b/configs/coins/groestlcoin_testnet.json index 46bc3248..24f9d1af 100644 --- a/configs/coins/groestlcoin_testnet.json +++ b/configs/coins/groestlcoin_testnet.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Groestlcoin Testnet", - "shortcut": "tGRS", - "label": "Groestlcoin Testnet", - "alias": "groestlcoin_testnet" + "name": "Groestlcoin Testnet", + "shortcut": "tGRS", + "label": "Groestlcoin Testnet", + "alias": "groestlcoin_testnet" }, "ports": { "backend_rpc": 18045, @@ -28,7 +28,7 @@ "verification_source": "f15bd5e38b25a103821f1563cd0e1b2cf7146ec9f9835493a30bd57313d3b86f", "extract_command": "mkdir -p backend/bin; tar -C backend/bin -xf", "exclude_files": [ - "bin/groestlcoin-qt" + "bin/groestlcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/groestlcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet3/*.log", @@ -40,8 +40,8 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "deprecatedrpc": "estimatefee", - "whitelist": "127.0.0.1" + "deprecatedrpc": "estimatefee", + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -56,6 +56,9 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 70617039, + "xpub_magic_segwit_p2sh": 71979618, + "xpub_magic_segwit_native": 73342198, "additional_params": {} } }, @@ -63,4 +66,4 @@ "package_maintainer": "Groestlcoin team", "package_maintainer_email": "support@groestlcoin.org" } -} +} \ No newline at end of file diff --git a/configs/coins/koto.json b/configs/coins/koto.json index c1581e52..8c0e4fd4 100644 --- a/configs/coins/koto.json +++ b/configs/coins/koto.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Koto", - "shortcut": "KOTO", - "label": "Koto", - "alias": "koto" + "name": "Koto", + "shortcut": "KOTO", + "label": "Koto", + "alias": "koto" }, "ports": { "backend_rpc": 8051, @@ -28,7 +28,7 @@ "verification_source": "https://github.com/KotoDevelopers/koto/releases/download/v2.0.2/koto-2.0.2-linux64-staticstdc++.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/koto-qt" + "bin/koto-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/kotod -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,9 +40,9 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "addnode": [ - "dnsseed.ko-to.org" - ] + "addnode": [ + "dnsseed.ko-to.org" + ] } }, "blockbook": { @@ -57,6 +57,7 @@ "mempool_workers": 4, "mempool_sub_workers": 8, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, "additional_params": {} } }, @@ -64,4 +65,4 @@ "package_maintainer": "WO", "package_maintainer_email": "wo@kotocoin.info" } -} +} \ No newline at end of file diff --git a/configs/coins/litecoin.json b/configs/coins/litecoin.json index 7368dca1..0ca821fb 100644 --- a/configs/coins/litecoin.json +++ b/configs/coins/litecoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Litecoin", - "shortcut": "LTC", - "label": "Litecoin", - "alias": "litecoin" + "name": "Litecoin", + "shortcut": "LTC", + "label": "Litecoin", + "alias": "litecoin" }, "ports": { "backend_rpc": 8034, @@ -28,7 +28,7 @@ "verification_source": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-linux-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/litecoin-qt" + "bin/litecoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/litecoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -55,6 +55,8 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 27108450, + "xpub_magic_segwit_p2sh": 28471030, "additional_params": {} } }, @@ -62,4 +64,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/litecoin_testnet.json b/configs/coins/litecoin_testnet.json index 767f0a62..96dc2559 100644 --- a/configs/coins/litecoin_testnet.json +++ b/configs/coins/litecoin_testnet.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Litecoin Testnet", - "shortcut": "TLTC", - "label": "Litecoin Testnet", - "alias": "litecoin_testnet" + "name": "Litecoin Testnet", + "shortcut": "TLTC", + "label": "Litecoin Testnet", + "alias": "litecoin_testnet" }, "ports": { "backend_rpc": 18034, @@ -28,7 +28,7 @@ "verification_source": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-linux-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/litecoin-qt" + "bin/litecoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/litecoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet4/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -55,6 +55,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 70617039, "additional_params": {} } }, @@ -62,4 +63,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/monacoin.json b/configs/coins/monacoin.json index d274e23f..d6c7b4d9 100644 --- a/configs/coins/monacoin.json +++ b/configs/coins/monacoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Monacoin", - "shortcut": "MONA", - "label": "Monacoin", - "alias": "monacoin" + "name": "Monacoin", + "shortcut": "MONA", + "label": "Monacoin", + "alias": "monacoin" }, "ports": { "backend_rpc": 8041, @@ -28,7 +28,7 @@ "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/monacoin-qt" + "bin/monacoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/monacoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -55,6 +55,8 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, "additional_params": {} } }, @@ -62,4 +64,4 @@ "package_maintainer": "wakiyamap", "package_maintainer_email": "wakiyamap@gmail.com" } -} +} \ No newline at end of file diff --git a/configs/coins/myriad.json b/configs/coins/myriad.json index 73398627..96f545f9 100644 --- a/configs/coins/myriad.json +++ b/configs/coins/myriad.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Myriad", - "shortcut": "XMY", - "label": "Myriad", - "alias": "myriad" + "name": "Myriad", + "shortcut": "XMY", + "label": "Myriad", + "alias": "myriad" }, "ports": { "backend_rpc": 8043, @@ -28,7 +28,7 @@ "verification_source": "a97b1e98a691666f6c2a61624050aebf7a894c4766cec031cae1ae7afba9a019", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/myriadcoin-qt" + "bin/myriadcoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/myriadcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,7 +40,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -55,6 +55,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, "additional_params": {} } }, @@ -62,4 +63,4 @@ "package_maintainer": "wlc-", "package_maintainer_email": "wwwwllllcccc@gmail.com" } -} +} \ No newline at end of file diff --git a/configs/coins/namecoin.json b/configs/coins/namecoin.json index 9d3f30e5..40b5a94e 100644 --- a/configs/coins/namecoin.json +++ b/configs/coins/namecoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Namecoin", - "shortcut": "NMC", - "label": "Namecoin", - "alias": "namecoin" + "name": "Namecoin", + "shortcut": "NMC", + "label": "Namecoin", + "alias": "namecoin" }, "ports": { "backend_rpc": 8039, @@ -28,7 +28,7 @@ "verification_source": "14ebaaf6f22f69b057a5bcb9b6959548f0a3f1b62cc113f19581d2297044827e", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ - "bin/namecoin-qt" + "bin/namecoin-qt" ], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/namecoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", @@ -40,14 +40,14 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1", - "upnp": 0, - "discover": 0, - "whitelistrelay": 1, - "listenonion": 0, - "addnode": [ - "45.24.110.177:8334" - ] + "addnode": [ + "45.24.110.177:8334" + ], + "discover": 0, + "listenonion": 0, + "upnp": 0, + "whitelist": "127.0.0.1", + "whitelistrelay": 1 } }, "blockbook": { @@ -62,6 +62,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, "additional_params": {} } }, @@ -69,4 +70,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/vertcoin.json b/configs/coins/vertcoin.json index 2f4c1b82..fab28ff2 100644 --- a/configs/coins/vertcoin.json +++ b/configs/coins/vertcoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Vertcoin", - "shortcut": "VTC", - "label": "Vertcoin", - "alias": "vertcoin" + "name": "Vertcoin", + "shortcut": "VTC", + "label": "Vertcoin", + "alias": "vertcoin" }, "ports": { "backend_rpc": 8040, @@ -38,7 +38,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "whitelist": "127.0.0.1" + "whitelist": "127.0.0.1" } }, "blockbook": { @@ -53,6 +53,8 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 1000, + "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, "additional_params": {} } }, @@ -60,4 +62,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index fc657d8f..234f1982 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Zcash", - "shortcut": "ZEC", - "label": "Zcash", - "alias": "zcash" + "name": "Zcash", + "shortcut": "ZEC", + "label": "Zcash", + "alias": "zcash" }, "ports": { "backend_rpc": 8032, @@ -38,9 +38,9 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "addnode": [ - "mainnet.z.cash" - ] + "addnode": [ + "mainnet.z.cash" + ] } }, "blockbook": { @@ -55,6 +55,7 @@ "mempool_workers": 4, "mempool_sub_workers": 8, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, "additional_params": {} } }, @@ -62,4 +63,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index 9df8ad31..88096df4 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Zcash Testnet", - "shortcut": "TAZ", - "label": "Zcash Testnet", - "alias": "zcash_testnet" + "name": "Zcash Testnet", + "shortcut": "TAZ", + "label": "Zcash Testnet", + "alias": "zcash_testnet" }, "ports": { "backend_rpc": 18032, @@ -38,9 +38,9 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "addnode": [ - "testnet.z.cash" - ] + "addnode": [ + "testnet.z.cash" + ] } }, "blockbook": { @@ -55,6 +55,7 @@ "mempool_workers": 4, "mempool_sub_workers": 8, "block_addresses_to_keep": 300, + "xpub_magic": 70617039, "additional_params": {} } }, @@ -62,4 +63,4 @@ "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/zcoin.json b/configs/coins/zcoin.json index a0e188a8..6ac38c25 100644 --- a/configs/coins/zcoin.json +++ b/configs/coins/zcoin.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Zcoin", - "shortcut": "XZC", - "label": "Zcoin", - "alias": "zcoin" + "name": "Zcoin", + "shortcut": "XZC", + "label": "Zcoin", + "alias": "zcoin" }, "ports": { "backend_rpc": 8050, @@ -52,7 +52,7 @@ "server_config_file": "bitcoin_like.conf", "client_config_file": "bitcoin_like_client.conf", "additional_params": { - "deprecatedrpc": "estimatefee" + "deprecatedrpc": "estimatefee" } }, "blockbook": { @@ -66,6 +66,7 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, + "xpub_magic": 76067358, "additional_params": {} } }, @@ -73,4 +74,4 @@ "package_maintainer": "Putta Khunchalee", "package_maintainer_email": "putta@zcoin.io" } -} +} \ No newline at end of file diff --git a/contrib/scripts/check-and-generate-port-registry.go b/contrib/scripts/check-and-generate-port-registry.go index 9e6039eb..9fcedcb1 100755 --- a/contrib/scripts/check-and-generate-port-registry.go +++ b/contrib/scripts/check-and-generate-port-registry.go @@ -237,7 +237,7 @@ func writeMarkdown(output string, slice PortInfoSlice) error { out := os.Stdout if output != "stdout" { - out, err = os.OpenFile(output, os.O_CREATE|os.O_WRONLY, 0644) + out, err = os.OpenFile(output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return err } diff --git a/docs/ports.md b/docs/ports.md index aae5e42e..e9f189cb 100644 --- a/docs/ports.md +++ b/docs/ports.md @@ -8,7 +8,7 @@ | Dash | 9033 | 9133 | 8033 | 38333 | | Litecoin | 9034 | 9134 | 8034 | 38334 | | Bitcoin Gold | 9035 | 9135 | 8035 | 38335 | -| Ethereum | 9036 | 9136 | 8036 | 38336 p2p, 8136 http | +| Ethereum | 9036 | 9136 | 8036 | 8136 http, 38336 p2p | | Ethereum Classic | 9037 | 9137 | 8037 | | | Dogecoin | 9038 | 9138 | 8038 | 38338 | | Namecoin | 9039 | 9139 | 8039 | 38339 | @@ -33,5 +33,6 @@ | Vertcoin Testnet | 19040 | 19140 | 18040 | 48340 | | Monacoin Testnet | 19041 | 19141 | 18041 | 48341 | | Groestlcoin Testnet | 19045 | 19145 | 18045 | 48345 | +| Koto Testnet | 19051 | 19151 | 18051 | 48351 | > NOTE: This document is generated from coin definitions in `configs/coins`. From 986275bb76bbc61b4886c7ebd393df8b2b70549f Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 28 Jan 2019 18:29:12 +0100 Subject: [PATCH 03/26] Implement parser.DeriveAddressDescriptors from xpub --- Gopkg.lock | 4 +- bchain/baseparser.go | 5 ++ bchain/coins/btc/bitcoinparser.go | 61 +++++++++++++++++++++- bchain/coins/btc/bitcoinparser_test.go | 72 ++++++++++++++++++++++++++ bchain/coins/btc/bitcoinrpc.go | 3 ++ bchain/types.go | 2 + 6 files changed, 144 insertions(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f26472e8..d0bea5ee 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -94,8 +94,8 @@ [[projects]] branch = "master" name = "github.com/martinboehm/btcutil" - packages = [".","base58","bech32","chaincfg","txscript"] - revision = "613fec26904062ae125fb073762af3a77c77b6c7" + packages = [".","base58","bech32","chaincfg","hdkeychain","txscript"] + revision = "9b332d8046124a83bab2830696e8ebddaf3f1788" [[projects]] branch = "master" diff --git a/bchain/baseparser.go b/bchain/baseparser.go index d18da3b4..e62b28f7 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -267,6 +267,11 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { return &tx, pt.Height, nil } +// DeriveAddressDescriptors is unsupported +func (p *BaseParser) DeriveAddressDescriptors(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) { + return nil, errors.New("Not supported") +} + // EthereumTypeGetErc20FromTx is unsupported func (p *BaseParser) EthereumTypeGetErc20FromTx(tx *Tx) ([]Erc20Transfer, error) { return nil, errors.New("Not supported") diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index f3731e0a..15264fd3 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -8,10 +8,12 @@ import ( "math/big" vlq "github.com/bsm/go-vlq" + "github.com/juju/errors" "github.com/martinboehm/btcd/blockchain" "github.com/martinboehm/btcd/wire" "github.com/martinboehm/btcutil" "github.com/martinboehm/btcutil/chaincfg" + "github.com/martinboehm/btcutil/hdkeychain" "github.com/martinboehm/btcutil/txscript" ) @@ -23,6 +25,9 @@ type BitcoinParser struct { *bchain.BaseParser Params *chaincfg.Params OutputScriptToAddressesFunc OutputScriptToAddressesFunc + XPubMagic uint32 + XPubMagicSegwitP2sh uint32 + XPubMagicSegwitNative uint32 } // NewBitcoinParser returns new BitcoinParser instance @@ -32,7 +37,10 @@ func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser BlockAddressesToKeep: c.BlockAddressesToKeep, AmountDecimalPoint: 8, }, - Params: params, + Params: params, + XPubMagic: c.XPubMagic, + XPubMagicSegwitP2sh: c.XPubMagicSegwitP2sh, + XPubMagicSegwitNative: c.XPubMagicSegwitNative, } p.OutputScriptToAddressesFunc = p.outputScriptToAddresses return p @@ -266,3 +274,54 @@ func (p *BitcoinParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { return tx, height, nil } + +func (p *BitcoinParser) addrDescFromExtKey(extKey *hdkeychain.ExtendedKey) (bchain.AddressDescriptor, error) { + var a btcutil.Address + var err error + if extKey.Version() == p.XPubMagicSegwitP2sh { + // redeemScript <20-byte-pubKeyHash> + pubKeyHash := btcutil.Hash160(extKey.PubKeyBytes()) + redeemScript := make([]byte, len(pubKeyHash)+2) + redeemScript[0] = 0 + redeemScript[1] = byte(len(pubKeyHash)) + copy(redeemScript[2:], pubKeyHash) + hash := btcutil.Hash160(redeemScript) + a, err = btcutil.NewAddressScriptHashFromHash(hash, p.Params) + } else if extKey.Version() == p.XPubMagicSegwitNative { + a, err = btcutil.NewAddressWitnessPubKeyHash(btcutil.Hash160(extKey.PubKeyBytes()), p.Params) + } else { + // default to P2PKH address + a, err = extKey.Address(p.Params) + } + if err != nil { + return nil, err + } + return txscript.PayToAddrScript(a) +} + +// DeriveAddressDescriptors derives address descriptors from given xpub +func (p *BitcoinParser) DeriveAddressDescriptors(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]bchain.AddressDescriptor, error) { + if toIndex <= fromIndex { + return nil, errors.New("toIndex<=fromIndex") + } + extKey, err := hdkeychain.NewKeyFromString(xpub) + if err != nil { + return nil, err + } + changeExtKey, err := extKey.Child(change) + if err != nil { + return nil, err + } + ad := make([]bchain.AddressDescriptor, toIndex-fromIndex) + for index := fromIndex; index < toIndex; index++ { + indexExtKey, err := changeExtKey.Child(index) + if err != nil { + return nil, err + } + ad[index-fromIndex], err = p.addrDescFromExtKey(indexExtKey) + if err != nil { + return nil, err + } + } + return ad, nil +} diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index e3062215..8aefdda0 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -417,3 +417,75 @@ func Test_UnpackTx(t *testing.T) { }) } } + +func Test_DeriveAddressDescriptors(t *testing.T) { + btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) + type args struct { + xpub string + change uint32 + fromIndex uint32 + toIndex uint32 + parser *BitcoinParser + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "m/44'/0'/0'", + args: args{ + xpub: "xpub6BosfCnifzxcFwrSzQiqu2DBVTshkCXacvNsWGYJVVhhawA7d4R5WSWGFNbi8Aw6ZRc1brxMyWMzG3DSSSSoekkudhUd9yLb6qx39T9nMdj", + change: 0, + fromIndex: 0, + toIndex: 1, + parser: btcMainParser, + }, + want: []string{"1LqBGSKuX5yYUonjxT5qGfpUsXKYYWeabA"}, + }, + { + name: "m/49'/0'/0'", + args: args{ + xpub: "ypub6Ww3ibxVfGzLrAH1PNcjyAWenMTbbAosGNB6VvmSEgytSER9azLDWCxoJwW7Ke7icmizBMXrzBx9979FfaHxHcrArf3zbeJJJUZPf663zsP", + change: 0, + fromIndex: 0, + toIndex: 1, + parser: btcMainParser, + }, + want: []string{"37VucYSaXLCAsxYyAPfbSi9eh4iEcbShgf"}, + }, + { + name: "m/84'/0'/0'", + args: args{ + xpub: "zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs", + change: 0, + fromIndex: 0, + toIndex: 1, + parser: btcMainParser, + }, + want: []string{"bc1qcr8te4kr609gcawutmrza0j4xv80jy8z306fyu"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.parser.DeriveAddressDescriptors(tt.args.xpub, tt.args.change, tt.args.fromIndex, tt.args.toIndex) + if (err != nil) != tt.wantErr { + t.Errorf("DeriveAddressDescriptors() error = %v, wantErr %v", err, tt.wantErr) + return + } + gotAddresses := make([]string, len(got)) + for i, ad := range got { + aa, _, err := tt.args.parser.GetAddressesFromAddrDesc(ad) + if err != nil || len(aa) != 1 { + t.Errorf("DeriveAddressDescriptors() got incorrect address descriptor %v, error %v", ad, err) + return + } + gotAddresses[i] = aa[0] + } + if !reflect.DeepEqual(gotAddresses, tt.want) { + t.Errorf("DeriveAddressDescriptors() = %v, want %v", gotAddresses, tt.want) + } + }) + } +} diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index a2fc0f11..e9f9e4ec 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -51,6 +51,9 @@ type Configuration struct { AddressFormat string `json:"address_format"` SupportsEstimateFee bool `json:"supports_estimate_fee"` SupportsEstimateSmartFee bool `json:"supports_estimate_smart_fee"` + XPubMagic uint32 `json:"xpub_magic,omitempty"` + XPubMagicSegwitP2sh uint32 `json:"xpub_magic_segwit_p2sh,omitempty"` + XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native,omitempty"` } // NewBitcoinRPC returns new BitcoinRPC instance. diff --git a/bchain/types.go b/bchain/types.go index a643b9dd..9f44940e 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -263,6 +263,8 @@ type BlockChainParser interface { PackBlockHash(hash string) ([]byte, error) UnpackBlockHash(buf []byte) (string, error) ParseBlock(b []byte) (*Block, error) + // xpub + DeriveAddressDescriptors(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) // EthereumType specific EthereumTypeGetErc20FromTx(tx *Tx) ([]Erc20Transfer, error) } From 27dba68319910850ff3d46429ceb303b429c8a3d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 29 Jan 2019 12:11:35 +0100 Subject: [PATCH 04/26] Add DeriveAddressDescriptors for list of indexes --- bchain/baseparser.go | 7 ++- bchain/coins/btc/bitcoinparser.go | 28 +++++++++- bchain/coins/btc/bitcoinparser_test.go | 76 ++++++++++++++++++++++++-- bchain/types.go | 3 +- 4 files changed, 106 insertions(+), 8 deletions(-) diff --git a/bchain/baseparser.go b/bchain/baseparser.go index e62b28f7..dbba33a5 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -268,7 +268,12 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } // DeriveAddressDescriptors is unsupported -func (p *BaseParser) DeriveAddressDescriptors(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) { +func (p *BaseParser) DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]AddressDescriptor, error) { + return nil, errors.New("Not supported") +} + +// DeriveAddressDescriptorsFromTo is unsupported +func (p *BaseParser) DeriveAddressDescriptorsFromTo(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) { return nil, errors.New("Not supported") } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 15264fd3..d4f13b02 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -299,8 +299,32 @@ func (p *BitcoinParser) addrDescFromExtKey(extKey *hdkeychain.ExtendedKey) (bcha return txscript.PayToAddrScript(a) } -// DeriveAddressDescriptors derives address descriptors from given xpub -func (p *BitcoinParser) DeriveAddressDescriptors(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]bchain.AddressDescriptor, error) { +// DeriveAddressDescriptors derives address descriptors from given xpub for listed indexes +func (p *BitcoinParser) DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]bchain.AddressDescriptor, error) { + extKey, err := hdkeychain.NewKeyFromString(xpub) + if err != nil { + return nil, err + } + changeExtKey, err := extKey.Child(change) + if err != nil { + return nil, err + } + ad := make([]bchain.AddressDescriptor, len(indexes)) + for i, index := range indexes { + indexExtKey, err := changeExtKey.Child(index) + if err != nil { + return nil, err + } + ad[i], err = p.addrDescFromExtKey(indexExtKey) + if err != nil { + return nil, err + } + } + return ad, nil +} + +// DeriveAddressDescriptorsFromTo derives address descriptors from given xpub for addresses in index range +func (p *BitcoinParser) DeriveAddressDescriptorsFromTo(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]bchain.AddressDescriptor, error) { if toIndex <= fromIndex { return nil, errors.New("toIndex<=fromIndex") } diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 8aefdda0..e610cca2 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -419,6 +419,74 @@ func Test_UnpackTx(t *testing.T) { } func Test_DeriveAddressDescriptors(t *testing.T) { + btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) + type args struct { + xpub string + change uint32 + indexes []uint32 + parser *BitcoinParser + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "m/44'/0'/0'", + args: args{ + xpub: "xpub6BosfCnifzxcFwrSzQiqu2DBVTshkCXacvNsWGYJVVhhawA7d4R5WSWGFNbi8Aw6ZRc1brxMyWMzG3DSSSSoekkudhUd9yLb6qx39T9nMdj", + change: 0, + indexes: []uint32{0, 1234}, + parser: btcMainParser, + }, + want: []string{"1LqBGSKuX5yYUonjxT5qGfpUsXKYYWeabA", "1P9w11dXAmG3QBjKLAvCsek8izs1iR2iFi"}, + }, + { + name: "m/49'/0'/0'", + args: args{ + xpub: "ypub6Ww3ibxVfGzLrAH1PNcjyAWenMTbbAosGNB6VvmSEgytSER9azLDWCxoJwW7Ke7icmizBMXrzBx9979FfaHxHcrArf3zbeJJJUZPf663zsP", + change: 0, + indexes: []uint32{0, 1234}, + parser: btcMainParser, + }, + want: []string{"37VucYSaXLCAsxYyAPfbSi9eh4iEcbShgf", "367meFzJ9KqDLm9PX6U8Z8RdmkSNBuxX8T"}, + }, + { + name: "m/84'/0'/0'", + args: args{ + xpub: "zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs", + change: 0, + indexes: []uint32{0, 1234}, + parser: btcMainParser, + }, + want: []string{"bc1qcr8te4kr609gcawutmrza0j4xv80jy8z306fyu", "bc1q4nm6g46ujzyjaeusralaz2nfv2rf04jjfyamkw"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.parser.DeriveAddressDescriptors(tt.args.xpub, tt.args.change, tt.args.indexes) + if (err != nil) != tt.wantErr { + t.Errorf("DeriveAddressDescriptorsFromTo() error = %v, wantErr %v", err, tt.wantErr) + return + } + gotAddresses := make([]string, len(got)) + for i, ad := range got { + aa, _, err := tt.args.parser.GetAddressesFromAddrDesc(ad) + if err != nil || len(aa) != 1 { + t.Errorf("DeriveAddressDescriptorsFromTo() got incorrect address descriptor %v, error %v", ad, err) + return + } + gotAddresses[i] = aa[0] + } + if !reflect.DeepEqual(gotAddresses, tt.want) { + t.Errorf("DeriveAddressDescriptorsFromTo() = %v, want %v", gotAddresses, tt.want) + } + }) + } +} + +func Test_DeriveAddressDescriptorsFromTo(t *testing.T) { btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) type args struct { xpub string @@ -469,22 +537,22 @@ func Test_DeriveAddressDescriptors(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.args.parser.DeriveAddressDescriptors(tt.args.xpub, tt.args.change, tt.args.fromIndex, tt.args.toIndex) + got, err := tt.args.parser.DeriveAddressDescriptorsFromTo(tt.args.xpub, tt.args.change, tt.args.fromIndex, tt.args.toIndex) if (err != nil) != tt.wantErr { - t.Errorf("DeriveAddressDescriptors() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("DeriveAddressDescriptorsFromTo() error = %v, wantErr %v", err, tt.wantErr) return } gotAddresses := make([]string, len(got)) for i, ad := range got { aa, _, err := tt.args.parser.GetAddressesFromAddrDesc(ad) if err != nil || len(aa) != 1 { - t.Errorf("DeriveAddressDescriptors() got incorrect address descriptor %v, error %v", ad, err) + t.Errorf("DeriveAddressDescriptorsFromTo() got incorrect address descriptor %v, error %v", ad, err) return } gotAddresses[i] = aa[0] } if !reflect.DeepEqual(gotAddresses, tt.want) { - t.Errorf("DeriveAddressDescriptors() = %v, want %v", gotAddresses, tt.want) + t.Errorf("DeriveAddressDescriptorsFromTo() = %v, want %v", gotAddresses, tt.want) } }) } diff --git a/bchain/types.go b/bchain/types.go index 9f44940e..f9e9245a 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -264,7 +264,8 @@ type BlockChainParser interface { UnpackBlockHash(buf []byte) (string, error) ParseBlock(b []byte) (*Block, error) // xpub - DeriveAddressDescriptors(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) + DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]AddressDescriptor, error) + DeriveAddressDescriptorsFromTo(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) // EthereumType specific EthereumTypeGetErc20FromTx(tx *Tx) ([]Erc20Transfer, error) } From b670b4fedee3479c17eb6616e01161086a6a0c6d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 30 Jan 2019 16:29:34 +0100 Subject: [PATCH 05/26] Add address derivation tests and benchmarks --- Gopkg.lock | 2 +- bchain/coins/btc/bitcoinparser_test.go | 47 ++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d0bea5ee..f1c9364c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -95,7 +95,7 @@ branch = "master" name = "github.com/martinboehm/btcutil" packages = [".","base58","bech32","chaincfg","hdkeychain","txscript"] - revision = "9b332d8046124a83bab2830696e8ebddaf3f1788" + revision = "520c2dbb6e0420531b6ae148845280378516e971" [[projects]] branch = "master" diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index e610cca2..f85aece1 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -19,7 +19,7 @@ func TestMain(m *testing.M) { os.Exit(c) } -func Test_GetAddrDescFromAddress(t *testing.T) { +func TestGetAddrDescFromAddress(t *testing.T) { type args struct { address string } @@ -77,7 +77,7 @@ func Test_GetAddrDescFromAddress(t *testing.T) { } } -func Test_GetAddrDescFromVout(t *testing.T) { +func TestGetAddrDescFromVout(t *testing.T) { type args struct { vout bchain.Vout } @@ -141,7 +141,7 @@ func Test_GetAddrDescFromVout(t *testing.T) { } } -func Test_GetAddressesFromAddrDesc(t *testing.T) { +func TestGetAddressesFromAddrDesc(t *testing.T) { type args struct { script string } @@ -316,7 +316,7 @@ func init() { } } -func Test_PackTx(t *testing.T) { +func TestPackTx(t *testing.T) { type args struct { tx bchain.Tx height uint32 @@ -367,7 +367,7 @@ func Test_PackTx(t *testing.T) { } } -func Test_UnpackTx(t *testing.T) { +func TestUnpackTx(t *testing.T) { type args struct { packedTx string parser *BitcoinParser @@ -418,7 +418,7 @@ func Test_UnpackTx(t *testing.T) { } } -func Test_DeriveAddressDescriptors(t *testing.T) { +func TestDeriveAddressDescriptors(t *testing.T) { btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) type args struct { xpub string @@ -486,8 +486,9 @@ func Test_DeriveAddressDescriptors(t *testing.T) { } } -func Test_DeriveAddressDescriptorsFromTo(t *testing.T) { +func TestDeriveAddressDescriptorsFromTo(t *testing.T) { btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) + btcTestnetsParser := NewBitcoinParser(GetChainParams("test"), &Configuration{XPubMagic: 70617039, XPubMagicSegwitP2sh: 71979618, XPubMagicSegwitNative: 73342198}) type args struct { xpub string change uint32 @@ -534,6 +535,17 @@ func Test_DeriveAddressDescriptorsFromTo(t *testing.T) { }, want: []string{"bc1qcr8te4kr609gcawutmrza0j4xv80jy8z306fyu"}, }, + { + name: "m/49'/1'/0'", + args: args{ + xpub: "upub5DR1Mg5nykixzYjFXWW5GghAU7dDqoPVJ2jrqFbL8sJ7Hs7jn69MP7KBnnmxn88GeZtnH8PRKV9w5MMSFX8AdEAoXY8Qd8BJPoXtpMeHMxJ", + change: 0, + fromIndex: 0, + toIndex: 10, + parser: btcTestnetsParser, + }, + want: []string{"2N4Q5FhU2497BryFfUgbqkAJE87aKHUhXMp", "2Mt7P2BAfE922zmfXrdcYTLyR7GUvbwSEns", "2N6aUMgQk8y1zvoq6FeWFyotyj75WY9BGsu", "2NA7tbZWM9BcRwBuebKSQe2xbhhF1paJwBM", "2N8RZMzvrUUnpLmvACX9ysmJ2MX3GK5jcQM", "2MvUUSiQZDSqyeSdofKX9KrSCio1nANPDTe", "2NBXaWu1HazjoUVgrXgcKNoBLhtkkD9Gmet", "2N791Ttf89tMVw2maj86E1Y3VgxD9Mc7PU7", "2NCJmwEq8GJm8t8GWWyBXAfpw7F2qZEVP5Y", "2NEgW71hWKer2XCSA8ZCC2VnWpB77L6bk68"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -557,3 +569,24 @@ func Test_DeriveAddressDescriptorsFromTo(t *testing.T) { }) } } + +func BenchmarkDeriveAddressDescriptorsFromToXpub(b *testing.B) { + btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) + for i := 0; i < b.N; i++ { + btcMainParser.DeriveAddressDescriptorsFromTo("xpub6BosfCnifzxcFwrSzQiqu2DBVTshkCXacvNsWGYJVVhhawA7d4R5WSWGFNbi8Aw6ZRc1brxMyWMzG3DSSSSoekkudhUd9yLb6qx39T9nMdj", 1, 0, 100) + } +} + +func BenchmarkDeriveAddressDescriptorsFromToYpub(b *testing.B) { + btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) + for i := 0; i < b.N; i++ { + btcMainParser.DeriveAddressDescriptorsFromTo("ypub6Ww3ibxVfGzLrAH1PNcjyAWenMTbbAosGNB6VvmSEgytSER9azLDWCxoJwW7Ke7icmizBMXrzBx9979FfaHxHcrArf3zbeJJJUZPf663zsP", 1, 0, 100) + } +} + +func BenchmarkDeriveAddressDescriptorsFromToZpub(b *testing.B) { + btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518}) + for i := 0; i < b.N; i++ { + btcMainParser.DeriveAddressDescriptorsFromTo("zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs", 1, 0, 100) + } +} From 225830d3e91b245b8b761d8ca09e39825969ece4 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 30 Jan 2019 17:56:15 +0100 Subject: [PATCH 06/26] Implement GetAddressForXpub and xpub explorer view - WIP --- api/types.go | 7 + api/xpub.go | 272 +++++++++++++++++++++++++++++++++++++ server/public.go | 69 ++++++++++ static/templates/base.html | 2 +- static/templates/xpub.html | 102 ++++++++++++++ 5 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 api/xpub.go create mode 100644 static/templates/xpub.html diff --git a/api/types.go b/api/types.go index cd1c7450..add7f7fd 100644 --- a/api/types.go +++ b/api/types.go @@ -5,6 +5,7 @@ import ( "blockbook/common" "blockbook/db" "encoding/json" + "errors" "math/big" "time" ) @@ -27,6 +28,9 @@ const ( TxHistory ) +// ErrUnsupportedXpub is returned when coin type does not support xpub address derivation or provided string is not an xpub +var ErrUnsupportedXpub = errors.New("XPUB not supported") + // APIError extends error by information if the error details should be returned to the end user type APIError struct { Text string @@ -123,6 +127,9 @@ type TokenType string // ERC20TokenType is Ethereum ERC20 token const ERC20TokenType TokenType = "ERC20" +// XPUBAddressTokenType is address derived from xpub +const XPUBAddressTokenType TokenType = "XPUBAddress" + // Token contains info about tokens held by an address type Token struct { Type TokenType `json:"type"` diff --git a/api/xpub.go b/api/xpub.go new file mode 100644 index 00000000..6c44e4fa --- /dev/null +++ b/api/xpub.go @@ -0,0 +1,272 @@ +package api + +import ( + "blockbook/bchain" + "blockbook/db" + "fmt" + "math/big" + "sync" + "time" + + "github.com/golang/glog" + "github.com/juju/errors" +) + +const xpubLen = 111 +const derivedAddressesBlock = 20 + +var cachedXpubs = make(map[string]*xpubData) +var cachedXpubsMux sync.Mutex + +type txHeight struct { + txid string + height uint32 + addrIndex uint32 +} + +type xpubAddress struct { + addrDesc bchain.AddressDescriptor + balance *db.AddrBalance + bottomHeight uint32 +} + +type xpubData struct { + dataHeight uint32 + dataHash string + txs uint32 + sentSat big.Int + balanceSat big.Int + addresses []xpubAddress + changeAddresses []xpubAddress + txids []txHeight +} + +func (w *Worker) getAddressTxHeights(addrDesc bchain.AddressDescriptor, addrIndex uint32, mempool bool, filter *AddressFilter, maxResults int) ([]txHeight, error) { + var err error + txHeights := make([]txHeight, 0, 4) + var callback db.GetTransactionsCallback + if filter.Vout == AddressFilterVoutOff { + callback = func(txid string, height uint32, indexes []int32) error { + txHeights = append(txHeights, txHeight{txid, height, addrIndex}) + // take all txs in the last found block even if it exceeds maxResults + if len(txHeights) >= maxResults && txHeights[len(txHeights)-1].height != height { + return &db.StopIteration{} + } + return nil + } + } else { + callback = func(txid string, height uint32, indexes []int32) error { + for _, index := range indexes { + vout := index + if vout < 0 { + vout = ^vout + } + if (filter.Vout == AddressFilterVoutInputs && index < 0) || + (filter.Vout == AddressFilterVoutOutputs && index >= 0) || + (vout == int32(filter.Vout)) { + txHeights = append(txHeights, txHeight{txid, height, addrIndex}) + if len(txHeights) >= maxResults { + return &db.StopIteration{} + } + break + } + } + return nil + } + } + if mempool { + uniqueTxs := make(map[string]struct{}) + o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) + if err != nil { + return nil, err + } + for _, m := range o { + if _, found := uniqueTxs[m.Txid]; !found { + l := len(txHeights) + callback(m.Txid, 0, []int32{m.Vout}) + if len(txHeights) > l { + uniqueTxs[m.Txid] = struct{}{} + } + } + } + } else { + to := filter.ToHeight + if to == 0 { + to = ^uint32(0) + } + err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback) + if err != nil { + return nil, err + } + } + return txHeights, nil +} + +func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, error) { + var err error + if ad.balance, err = w.db.GetAddrDescBalance(ad.addrDesc); err != nil { + return false, err + } + if ad.balance != nil { + data.txs += ad.balance.Txs + data.sentSat.Add(&data.sentSat, &ad.balance.SentSat) + data.balanceSat.Add(&data.balanceSat, &ad.balance.BalanceSat) + return true, nil + } + return false, nil +} + +func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token { + a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) + var address string + if len(a) > 0 { + address = a[0] + } + return Token{ + Type: XPUBAddressTokenType, + Name: address, + Decimals: w.chainParser.AmountDecimals(), + BalanceSat: (*Amount)(&ad.balance.BalanceSat), + Transfers: int(ad.balance.Txs), + Contract: fmt.Sprintf("%d/%d", changeIndex, index), + } +} + +// GetAddressForXpub computes address value and gets transactions for given address +func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { + if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { + return nil, ErrUnsupportedXpub + } + start := time.Now() + var processedHash string + cachedXpubsMux.Lock() + data, found := cachedXpubs[xpub] + cachedXpubsMux.Unlock() + // to load all data for xpub may take some time, perform it in a loop to process a possible new block + for { + bestheight, besthash, err := w.db.GetBestBlock() + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + if besthash == processedHash { + break + } + fork := false + if !found { + data = &xpubData{} + } else { + hash, err := w.db.GetBlockHash(data.dataHeight) + if err != nil { + return nil, err + } + if hash != data.dataHash { + // in case of for reset all cached txids + fork = true + data.txids = nil + } + } + processedHash = besthash + if data.dataHeight < bestheight { + data.dataHeight = bestheight + data.dataHash = besthash + // rescan known addresses + lastUsed := 0 + for i := range data.addresses { + ad := &data.addresses[i] + if fork { + ad.bottomHeight = 0 + } + used, err := w.derivedAddressBalance(data, ad) + if err != nil { + return nil, err + } + if used { + lastUsed = i + } + } + // derive new addresses as necessary + missing := len(data.addresses) - lastUsed + for missing < derivedAddressesBlock { + from := len(data.addresses) + descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, 0, uint32(from), uint32(from+derivedAddressesBlock-missing)) + if err != nil { + return nil, err + } + for i, a := range descriptors { + ad := xpubAddress{addrDesc: a} + used, err := w.derivedAddressBalance(data, &ad) + if err != nil { + return nil, err + } + if used { + lastUsed = i + from + } + data.addresses = append(data.addresses, ad) + } + missing = len(data.addresses) - lastUsed + } + // check and generate change addresses + ca := data.changeAddresses + data.changeAddresses = make([]xpubAddress, len(data.addresses)) + copy(data.changeAddresses, ca) + changeIndexes := []uint32{} + for i, ad := range data.addresses { + if ad.balance != nil { + if data.changeAddresses[i].addrDesc == nil { + changeIndexes = append(changeIndexes, uint32(i)) + } else { + _, err := w.derivedAddressBalance(data, &ad) + if err != nil { + return nil, err + } + } + } + } + if len(changeIndexes) > 0 { + descriptors, err := w.chainParser.DeriveAddressDescriptors(xpub, 1, changeIndexes) + if err != nil { + return nil, err + } + for i, a := range descriptors { + ad := &data.changeAddresses[changeIndexes[i]] + ad.addrDesc = a + _, err := w.derivedAddressBalance(data, ad) + if err != nil { + return nil, err + } + } + } + } + } + cachedXpubsMux.Lock() + cachedXpubs[xpub] = data + cachedXpubsMux.Unlock() + tokens := make([]Token, 0, 4) + for i, ad := range data.addresses { + if ad.balance != nil { + tokens = append(tokens, w.tokenFromXpubAddress(&ad, 0, i)) + } + if data.changeAddresses[i].balance != nil { + tokens = append(tokens, w.tokenFromXpubAddress(&data.changeAddresses[i], 1, i)) + } + } + var totalReceived big.Int + totalReceived.Add(&data.balanceSat, &data.sentSat) + addr := Address{ + // Paging: pg, + AddrStr: xpub, + BalanceSat: (*Amount)(&data.balanceSat), + TotalReceivedSat: (*Amount)(&totalReceived), + TotalSentSat: (*Amount)(&data.sentSat), + Txs: int(data.txs), + // UnconfirmedBalanceSat: (*Amount)(&uBalSat), + // UnconfirmedTxs: len(txm), + // Transactions: txs, + // Txids: txids, + Tokens: tokens, + // Erc20Contract: erc20c, + // Nonce: nonce, + } + glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) + return &addr, nil +} diff --git a/server/public.go b/server/public.go index a18b3efd..02c79120 100644 --- a/server/public.go +++ b/server/public.go @@ -131,6 +131,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { // internal explorer handlers serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx)) serveMux.HandleFunc(path+"address/", s.htmlTemplateHandler(s.explorerAddress)) + serveMux.HandleFunc(path+"xpub/", s.htmlTemplateHandler(s.explorerXpub)) serveMux.HandleFunc(path+"search/", s.htmlTemplateHandler(s.explorerSearch)) serveMux.HandleFunc(path+"blocks", s.htmlTemplateHandler(s.explorerBlocks)) serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock)) @@ -165,6 +166,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiDefault)) serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx, apiDefault)) serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress, apiDefault)) + serveMux.HandleFunc(path+"api/xpub/", s.jsonHandler(s.apiXpub, apiDefault)) serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo, apiDefault)) serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock, apiDefault)) serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx, apiDefault)) @@ -174,6 +176,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/v2/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV2)) serveMux.HandleFunc(path+"api/v2/tx/", s.jsonHandler(s.apiTx, apiV2)) serveMux.HandleFunc(path+"api/v2/address/", s.jsonHandler(s.apiAddress, apiV2)) + serveMux.HandleFunc(path+"api/v2/xpub/", s.jsonHandler(s.apiXpub, apiV2)) serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV2)) serveMux.HandleFunc(path+"api/v2/block/", s.jsonHandler(s.apiBlock, apiV2)) serveMux.HandleFunc(path+"api/v2/sendtx/", s.jsonHandler(s.apiSendTx, apiV2)) @@ -372,6 +375,7 @@ const ( indexTpl txTpl addressTpl + xpubTpl blocksTpl blockTpl sendTransactionTpl @@ -465,6 +469,7 @@ func (s *PublicServer) parseTemplates() []*template.Template { t[addressTpl] = createTemplate("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") t[blockTpl] = createTemplate("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") } + t[xpubTpl] = createTemplate("./static/templates/xpub.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") return t } @@ -577,6 +582,48 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( return addressTpl, data, nil } +func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var address *api.Address + var filter string + var fn = api.AddressFilterVoutOff + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + filter = r.URL.Query().Get("filter") + if len(filter) > 0 { + if filter == "inputs" { + fn = api.AddressFilterVoutInputs + } else if filter == "outputs" { + fn = api.AddressFilterVoutOutputs + } else { + fn, ec = strconv.Atoi(filter) + if ec != nil || fn < 0 { + filter = "" + fn = api.AddressFilterVoutOff + } + } + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}) + if err != nil { + return errorTpl, nil, err + } + } + data := s.newTemplateData() + data.AddrStr = address.AddrStr + data.Address = address + data.Page = address.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + if filter != "" { + data.PageParams = template.URL("&filter=" + filter) + data.Address.Filter = filter + } + return xpubTpl, data, nil +} + func (s *PublicServer) explorerBlocks(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var blocks *api.Blocks var err error @@ -638,6 +685,11 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t var err error s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() if len(q) > 0 { + address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + if err == nil { + http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302) + return noTpl, nil, nil + } block, err = s.api.GetBlock(q, 0, 1) if err == nil { http.Redirect(w, r, joinURL("/block/", block.Hash), 302) @@ -818,6 +870,23 @@ func (s *PublicServer) apiAddress(r *http.Request, apiVersion int) (interface{}, return address, err } +func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, error) { + var address *api.Address + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + if err == nil && apiVersion == apiV1 { + return s.api.AddressToV1(address), nil + } + } + return address, err +} + func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interface{}, error) { var utxo []api.AddressUtxo var err error diff --git a/static/templates/base.html b/static/templates/base.html index bc1e9f02..cecc88d9 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -42,7 +42,7 @@ {{- end -}} diff --git a/static/templates/xpub.html b/static/templates/xpub.html new file mode 100644 index 00000000..7dbebf46 --- /dev/null +++ b/static/templates/xpub.html @@ -0,0 +1,102 @@ +{{define "specific"}}{{$cs := .CoinShortcut}}{{$addr := .Address}}{{$data := .}} +

XPUB {{formatAmount $addr.BalanceSat}} {{$cs}} +

+
+ {{$addr.AddrStr}} +
+

Confirmed

+
+
+ + + + + + + + + + + + + + + + + + + {{- if $addr.Tokens -}} + + + + + {{- end -}} + +
Total Received{{formatAmount $addr.TotalReceivedSat}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSentSat}} {{$cs}}
Final Balance{{formatAmount $addr.BalanceSat}} {{$cs}}
No. Transactions{{$addr.Txs}}
XPUB addresses + + + + + + + + + {{- range $t := $addr.Tokens -}} + + + + + + + {{- end -}} + +
AddressBalanceTxsPath
{{$t.Name}}{{formatAmount $t.BalanceSat}} {{$cs}}{{$t.Transfers}}{{$t.Contract}}
+
+
+
+
+ + +
+
+{{- if $addr.UnconfirmedTxs -}} +

Unconfirmed

+
+ + + + + + + + + + + +
Unconfirmed Balance{{formatAmount $addr.UnconfirmedBalanceSat}} {{$cs}}
No. Transactions{{$addr.UnconfirmedTxs}}
+
+{{- end}}{{if or $addr.Transactions $addr.Filter -}} +
+

Transactions

+ +
+ +
+
+
+ {{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}} +
+ +{{end}}{{end}} \ No newline at end of file From 8f1f1c87ace0287dfd967b320bfe931b32a299e2 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 31 Jan 2019 08:30:18 +0100 Subject: [PATCH 07/26] Modify the discovery of xpub addresses --- api/xpub.go | 144 ++++++++++++++++++------------------- server/public.go | 14 +++- static/templates/xpub.html | 4 +- 3 files changed, 83 insertions(+), 79 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index 6c44e4fa..5c35dc1a 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -13,7 +13,7 @@ import ( ) const xpubLen = 111 -const derivedAddressesBlock = 20 +const defaultAddressesGap = 20 var cachedXpubs = make(map[string]*xpubData) var cachedXpubsMux sync.Mutex @@ -31,6 +31,7 @@ type xpubAddress struct { } type xpubData struct { + gap int dataHeight uint32 dataHash string txs uint32 @@ -116,6 +117,50 @@ func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, e return false, nil } +func (w *Worker) scanAddresses(xpub string, data *xpubData, addresses []xpubAddress, gap int, change int, minDerivedIndex int, fork bool) (int, []xpubAddress, error) { + // rescan known addresses + lastUsed := 0 + for i := range addresses { + ad := &addresses[i] + if fork { + ad.bottomHeight = 0 + } + used, err := w.derivedAddressBalance(data, ad) + if err != nil { + return 0, nil, err + } + if used { + lastUsed = i + } + } + // derive new addresses as necessary + missing := len(addresses) - lastUsed + for missing < gap { + from := len(addresses) + to := from + gap - missing + if to < minDerivedIndex { + to = minDerivedIndex + } + descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, uint32(change), uint32(from), uint32(to)) + if err != nil { + return 0, nil, err + } + for i, a := range descriptors { + ad := xpubAddress{addrDesc: a} + used, err := w.derivedAddressBalance(data, &ad) + if err != nil { + return 0, nil, err + } + if used { + lastUsed = i + from + } + addresses = append(addresses, ad) + } + missing = len(addresses) - lastUsed + } + return lastUsed, addresses, nil +} + func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token { a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) var address string @@ -133,11 +178,16 @@ func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index in } // GetAddressForXpub computes address value and gets transactions for given address -func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { +func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*Address, error) { if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { return nil, ErrUnsupportedXpub } start := time.Now() + if gap <= 0 { + gap = defaultAddressesGap + } + // gap is increased one as there must be gap of empty addresses before the derivation is stopped + gap++ var processedHash string cachedXpubsMux.Lock() data, found := cachedXpubs[xpub] @@ -152,8 +202,8 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option break } fork := false - if !found { - data = &xpubData{} + if !found || data.gap != gap { + data = &xpubData{gap: gap} } else { hash, err := w.db.GetBlockHash(data.dataHeight) if err != nil { @@ -169,72 +219,14 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if data.dataHeight < bestheight { data.dataHeight = bestheight data.dataHash = besthash - // rescan known addresses - lastUsed := 0 - for i := range data.addresses { - ad := &data.addresses[i] - if fork { - ad.bottomHeight = 0 - } - used, err := w.derivedAddressBalance(data, ad) - if err != nil { - return nil, err - } - if used { - lastUsed = i - } + var lastUsedIndex int + lastUsedIndex, data.addresses, err = w.scanAddresses(xpub, data, data.addresses, gap, 0, 0, fork) + if err != nil { + return nil, err } - // derive new addresses as necessary - missing := len(data.addresses) - lastUsed - for missing < derivedAddressesBlock { - from := len(data.addresses) - descriptors, err := w.chainParser.DeriveAddressDescriptorsFromTo(xpub, 0, uint32(from), uint32(from+derivedAddressesBlock-missing)) - if err != nil { - return nil, err - } - for i, a := range descriptors { - ad := xpubAddress{addrDesc: a} - used, err := w.derivedAddressBalance(data, &ad) - if err != nil { - return nil, err - } - if used { - lastUsed = i + from - } - data.addresses = append(data.addresses, ad) - } - missing = len(data.addresses) - lastUsed - } - // check and generate change addresses - ca := data.changeAddresses - data.changeAddresses = make([]xpubAddress, len(data.addresses)) - copy(data.changeAddresses, ca) - changeIndexes := []uint32{} - for i, ad := range data.addresses { - if ad.balance != nil { - if data.changeAddresses[i].addrDesc == nil { - changeIndexes = append(changeIndexes, uint32(i)) - } else { - _, err := w.derivedAddressBalance(data, &ad) - if err != nil { - return nil, err - } - } - } - } - if len(changeIndexes) > 0 { - descriptors, err := w.chainParser.DeriveAddressDescriptors(xpub, 1, changeIndexes) - if err != nil { - return nil, err - } - for i, a := range descriptors { - ad := &data.changeAddresses[changeIndexes[i]] - ad.addrDesc = a - _, err := w.derivedAddressBalance(data, ad) - if err != nil { - return nil, err - } - } + _, data.changeAddresses, err = w.scanAddresses(xpub, data, data.changeAddresses, gap, 1, lastUsedIndex, fork) + if err != nil { + return nil, err } } } @@ -242,12 +234,16 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubs[xpub] = data cachedXpubsMux.Unlock() tokens := make([]Token, 0, 4) - for i, ad := range data.addresses { + for i := range data.addresses { + ad := &data.addresses[i] if ad.balance != nil { - tokens = append(tokens, w.tokenFromXpubAddress(&ad, 0, i)) + tokens = append(tokens, w.tokenFromXpubAddress(ad, 0, i)) } - if data.changeAddresses[i].balance != nil { - tokens = append(tokens, w.tokenFromXpubAddress(&data.changeAddresses[i], 1, i)) + } + for i := range data.changeAddresses { + ad := &data.changeAddresses[i] + if ad.balance != nil { + tokens = append(tokens, w.tokenFromXpubAddress(ad, 1, i)) } } var totalReceived big.Int @@ -267,6 +263,6 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option // Erc20Contract: erc20c, // Nonce: nonce, } - glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) + glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) return &addr, nil } diff --git a/server/public.go b/server/public.go index 02c79120..a993cf63 100644 --- a/server/public.go +++ b/server/public.go @@ -607,7 +607,11 @@ func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl } } } - address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}) + gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) + if ec != nil { + gap = 0 + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}, gap) if err != nil { return errorTpl, nil, err } @@ -685,7 +689,7 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t var err error s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() if len(q) > 0 { - address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, 0) if err == nil { http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302) return noTpl, nil, nil @@ -879,7 +883,11 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er if ec != nil { page = 0 } - address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) + if ec != nil { + gap = 0 + } + address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, gap) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 7dbebf46..94b9a283 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -34,8 +34,8 @@ Address Balance - Txs - Path + Txs + Path {{- range $t := $addr.Tokens -}} From 57b40ad6dca5b2d788d98dc204362516c0cf0aa0 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 31 Jan 2019 15:04:09 +0100 Subject: [PATCH 08/26] Hide xpub addresses with zero balance in explorer by default --- api/types.go | 7 ++++ api/worker.go | 2 +- api/xpub.go | 14 ++++++-- server/public.go | 68 +++++++++++++++++++------------------- static/templates/xpub.html | 11 +++++- 5 files changed, 63 insertions(+), 39 deletions(-) diff --git a/api/types.go b/api/types.go index add7f7fd..75d26598 100644 --- a/api/types.go +++ b/api/types.go @@ -52,6 +52,11 @@ func NewAPIError(s string, public bool) error { // Amount is datatype holding amounts type Amount big.Int +// IsZeroBigInt if big int has zero value +func IsZeroBigInt(b *big.Int) bool { + return len(b.Bits()) == 0 +} + // MarshalJSON Amount serialization func (a *Amount) MarshalJSON() (out []byte, err error) { if a == nil { @@ -207,6 +212,7 @@ type AddressFilter struct { Contract string FromHeight uint32 ToHeight uint32 + AllTokens bool } // Address holds information about address and its transactions @@ -223,6 +229,7 @@ type Address struct { Transactions []*Tx `json:"transactions,omitempty"` Txids []string `json:"txids,omitempty"` Nonce string `json:"nonce,omitempty"` + TotalTokens int `json:"totalTokens,omitempty"` Tokens []Token `json:"tokens,omitempty"` Erc20Contract *bchain.Erc20Contract `json:"erc20contract,omitempty"` Filter string `json:"-"` diff --git a/api/worker.go b/api/worker.go index 9318ef5a..2acd69ae 100644 --- a/api/worker.go +++ b/api/worker.go @@ -825,7 +825,7 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt } var checksum big.Int // ba can be nil if the address is only in mempool! - if ba != nil && ba.BalanceSat.Uint64() > 0 { + if ba != nil && !IsZeroBigInt(&ba.BalanceSat) { outpoints := make([]bchain.Outpoint, 0, 8) err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, height uint32, indexes []int32) error { for _, index := range indexes { diff --git a/api/xpub.go b/api/xpub.go index 5c35dc1a..19081344 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -233,17 +233,24 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubsMux.Lock() cachedXpubs[xpub] = data cachedXpubsMux.Unlock() + totalTokens := 0 tokens := make([]Token, 0, 4) for i := range data.addresses { ad := &data.addresses[i] if ad.balance != nil { - tokens = append(tokens, w.tokenFromXpubAddress(ad, 0, i)) + totalTokens++ + if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { + tokens = append(tokens, w.tokenFromXpubAddress(ad, 0, i)) + } } } for i := range data.changeAddresses { ad := &data.changeAddresses[i] if ad.balance != nil { - tokens = append(tokens, w.tokenFromXpubAddress(ad, 1, i)) + totalTokens++ + if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { + tokens = append(tokens, w.tokenFromXpubAddress(ad, 1, i)) + } } } var totalReceived big.Int @@ -259,7 +266,8 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option // UnconfirmedTxs: len(txm), // Transactions: txs, // Txids: txids, - Tokens: tokens, + TotalTokens: totalTokens, + Tokens: tokens, // Erc20Contract: erc20c, // Nonce: nonce, } diff --git a/server/public.go b/server/public.go index a993cf63..d40567c3 100644 --- a/server/public.go +++ b/server/public.go @@ -405,6 +405,7 @@ type TemplateData struct { TOSLink string SendTxHex string Status string + AllTokens bool } func (s *PublicServer) parseTemplates() []*template.Template { @@ -582,36 +583,40 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( return addressTpl, data, nil } +func (s *PublicServer) getAddressForXpub(r *http.Request, xpub string, pageSize int, option api.GetAddressOption) (*api.Address, error) { + var fn = api.AddressFilterVoutOff + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + filter := r.URL.Query().Get("filter") + if len(filter) > 0 { + if filter == "inputs" { + fn = api.AddressFilterVoutInputs + } else if filter == "outputs" { + fn = api.AddressFilterVoutOutputs + } else { + fn, ec = strconv.Atoi(filter) + if ec != nil || fn < 0 { + filter = "" + fn = api.AddressFilterVoutOff + } + } + } + gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) + if ec != nil { + gap = 0 + } + allAddresses, _ := strconv.ParseBool(r.URL.Query().Get("alladdresses")) + return s.api.GetAddressForXpub(xpub, page, pageSize, option, &api.AddressFilter{Vout: fn, AllTokens: allAddresses}, gap) +} + func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var address *api.Address - var filter string - var fn = api.AddressFilterVoutOff var err error s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - page, ec := strconv.Atoi(r.URL.Query().Get("page")) - if ec != nil { - page = 0 - } - filter = r.URL.Query().Get("filter") - if len(filter) > 0 { - if filter == "inputs" { - fn = api.AddressFilterVoutInputs - } else if filter == "outputs" { - fn = api.AddressFilterVoutOutputs - } else { - fn, ec = strconv.Atoi(filter) - if ec != nil || fn < 0 { - filter = "" - fn = api.AddressFilterVoutOff - } - } - } - gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) - if ec != nil { - gap = 0 - } - address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}, gap) + address, err = s.getAddressForXpub(r, r.URL.Path[i+1:], txsOnPage, api.TxHistoryLight) if err != nil { return errorTpl, nil, err } @@ -621,10 +626,13 @@ func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl data.Address = address data.Page = address.Page data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + filter := r.URL.Query().Get("filter") if filter != "" { data.PageParams = template.URL("&filter=" + filter) data.Address.Filter = filter } + allAddresses := r.URL.Query().Get("alladdresses") + data.AllTokens, _ = strconv.ParseBool(allAddresses) return xpubTpl, data, nil } @@ -879,15 +887,7 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - page, ec := strconv.Atoi(r.URL.Query().Get("page")) - if ec != nil { - page = 0 - } - gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) - if ec != nil { - gap = 0 - } - address, err = s.api.GetAddressForXpub(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, gap) + address, err = s.getAddressForXpub(r, r.URL.Path[i+1:], txsInAPI, api.TxidHistory) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 94b9a283..059e32a3 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -25,9 +25,13 @@ No. Transactions {{$addr.Txs}} + + Total XPUB addresses + {{$addr.TotalTokens}} + {{- if $addr.Tokens -}} - XPUB addresses + {{if $data.AllTokens}}XPUB Addresses{{else}}Nonzero XPUB Addresses{{end}} @@ -45,6 +49,11 @@ {{- end -}} + {{- if not $data.AllTokens -}} + + + + {{- end -}}
{{$t.Contract}}
Show all XPUB addresses
From 9d3cd3b3e96bff0dcdd235acd665a6222e42ccd4 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 3 Feb 2019 23:42:44 +0100 Subject: [PATCH 09/26] Implement loading of transactions in GetAddressForXpub - WIP --- api/types.go | 1 + api/worker.go | 61 +++++---- api/xpub.go | 270 ++++++++++++++++++++++++++++--------- static/templates/xpub.html | 10 +- 4 files changed, 241 insertions(+), 101 deletions(-) diff --git a/api/types.go b/api/types.go index 75d26598..6cea9243 100644 --- a/api/types.go +++ b/api/types.go @@ -10,6 +10,7 @@ import ( "time" ) +const maxUint32 = ^uint32(0) const maxInt = int(^uint(0) >> 1) // GetAddressOption specifies what data returns GetAddress api call diff --git a/api/worker.go b/api/worker.go index 2acd69ae..d014db09 100644 --- a/api/worker.go +++ b/api/worker.go @@ -582,6 +582,39 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto return ba, tokens, ci, n, nonContractTxs, totalResults, nil } +func (w *Worker) txFromTxid(txid string, bestheight uint32, option GetAddressOption) (*Tx, error) { + var tx *Tx + var err error + // only ChainBitcoinType supports TxHistoryLight + if option == TxHistoryLight && w.chainType == bchain.ChainBitcoinType { + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + // as fallback, provide empty TxAddresses to return at least something + ta = &db.TxAddresses{} + } + bi, err := w.db.GetBlockInfo(ta.Height) + if err != nil { + return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) + } + if bi == nil { + glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") + // provide empty BlockInfo to return the rest of tx data + bi = &db.BlockInfo{} + } + tx = w.txFromTxAddress(txid, ta, bi, bestheight) + } else { + tx, err = w.GetTransaction(txid, false, true) + if err != nil { + return nil, errors.Annotatef(err, "GetTransaction %v", txid) + } + } + return tx, nil +} + // GetAddress computes address value and gets transactions for given address func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { start := time.Now() @@ -703,32 +736,8 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA if option == TxidHistory { txids[txi] = txid } else { - // only ChainBitcoinType supports TxHistoryLight - if option == TxHistoryLight && w.chainType == bchain.ChainBitcoinType { - ta, err := w.db.GetTxAddresses(txid) - if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) - } - if ta == nil { - glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") - // as fallback, provide empty TxAddresses to return at least something - ta = &db.TxAddresses{} - } - bi, err := w.db.GetBlockInfo(ta.Height) - if err != nil { - return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) - } - if bi == nil { - glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") - // provide empty BlockInfo to return the rest of tx data - bi = &db.BlockInfo{} - } - txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) - } else { - txs[txi], err = w.GetTransaction(txid, false, true) - if err != nil { - return nil, errors.Annotatef(err, "GetTransaction %v", txid) - } + if txs[txi], err = w.txFromTxid(txid, bestheight, option); err != nil { + return nil, err } } txi++ diff --git a/api/xpub.go b/api/xpub.go index 19081344..141b8381 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -5,6 +5,7 @@ import ( "blockbook/db" "fmt" "math/big" + "sort" "sync" "time" @@ -15,19 +16,31 @@ import ( const xpubLen = 111 const defaultAddressesGap = 20 +const txInput = 1 +const txOutput = 2 + var cachedXpubs = make(map[string]*xpubData) var cachedXpubsMux sync.Mutex -type txHeight struct { - txid string - height uint32 - addrIndex uint32 +type xpubTxid struct { + txid string + height uint32 + inputOutput byte } +type xpubTxids []xpubTxid + +func (a xpubTxids) Len() int { return len(a) } +func (a xpubTxids) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a xpubTxids) Less(i, j int) bool { return a[i].height >= a[j].height } + type xpubAddress struct { - addrDesc bchain.AddressDescriptor - balance *db.AddrBalance - bottomHeight uint32 + addrDesc bchain.AddressDescriptor + balance *db.AddrBalance + txs uint32 + maxHeight uint32 + complete bool + txids xpubTxids } type xpubData struct { @@ -39,71 +52,101 @@ type xpubData struct { balanceSat big.Int addresses []xpubAddress changeAddresses []xpubAddress - txids []txHeight } -func (w *Worker) getAddressTxHeights(addrDesc bchain.AddressDescriptor, addrIndex uint32, mempool bool, filter *AddressFilter, maxResults int) ([]txHeight, error) { +func (w *Worker) xpubGetAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool, fromHeight, toHeight uint32, maxResults int) ([]xpubTxid, bool, error) { var err error - txHeights := make([]txHeight, 0, 4) + complete := true + txs := make([]xpubTxid, 0, 4) var callback db.GetTransactionsCallback - if filter.Vout == AddressFilterVoutOff { - callback = func(txid string, height uint32, indexes []int32) error { - txHeights = append(txHeights, txHeight{txid, height, addrIndex}) - // take all txs in the last found block even if it exceeds maxResults - if len(txHeights) >= maxResults && txHeights[len(txHeights)-1].height != height { - return &db.StopIteration{} - } - return nil + callback = func(txid string, height uint32, indexes []int32) error { + // take all txs in the last found block even if it exceeds maxResults + if len(txs) >= maxResults && txs[len(txs)-1].height != height { + complete = false + return &db.StopIteration{} } - } else { - callback = func(txid string, height uint32, indexes []int32) error { - for _, index := range indexes { - vout := index - if vout < 0 { - vout = ^vout - } - if (filter.Vout == AddressFilterVoutInputs && index < 0) || - (filter.Vout == AddressFilterVoutOutputs && index >= 0) || - (vout == int32(filter.Vout)) { - txHeights = append(txHeights, txHeight{txid, height, addrIndex}) - if len(txHeights) >= maxResults { - return &db.StopIteration{} - } - break - } + inputOutput := byte(0) + for _, index := range indexes { + if index < 0 { + inputOutput |= txInput + } else { + inputOutput |= txOutput } - return nil } + txs = append(txs, xpubTxid{txid, height, inputOutput}) + return nil } if mempool { - uniqueTxs := make(map[string]struct{}) + uniqueTxs := make(map[string]int) o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) if err != nil { - return nil, err + return nil, false, err } for _, m := range o { - if _, found := uniqueTxs[m.Txid]; !found { - l := len(txHeights) + if l, found := uniqueTxs[m.Txid]; !found { + l = len(txs) callback(m.Txid, 0, []int32{m.Vout}) - if len(txHeights) > l { - uniqueTxs[m.Txid] = struct{}{} + if len(txs) > l { + uniqueTxs[m.Txid] = l - 1 + } + } else { + if m.Vout < 0 { + txs[l].inputOutput |= txInput + } else { + txs[l].inputOutput |= txOutput } } } } else { - to := filter.ToHeight - if to == 0 { - to = ^uint32(0) - } - err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback) + err = w.db.GetAddrDescTransactions(addrDesc, fromHeight, toHeight, callback) if err != nil { - return nil, err + return nil, false, err } } - return txHeights, nil + return txs, complete, nil } -func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, error) { +func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, maxHeight uint32, pageSize int) error { + // skip if not discovered + if ad.balance == nil { + return nil + } + // if completely read, check if there are not some new txs and load if necessary + if ad.complete { + if ad.balance.Txs != ad.txs { + newTxids, _, err := w.xpubGetAddressTxids(ad.addrDesc, false, ad.maxHeight+1, maxHeight, maxInt) + if err == nil { + ad.txids = append(newTxids, ad.txids...) + ad.maxHeight = maxHeight + ad.txs = uint32(len(ad.txids)) + if ad.txs != ad.balance.Txs { + glog.Warning("xpubCheckAndLoadTxids inconsistency ", ad.addrDesc, ", ad.txs=", ad.txs, ", ad.balance.Txs=", ad.balance.Txs) + } + } + return err + } + } + // unless the filter is completely off, load all txids + if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { + pageSize = maxInt + } + newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, 0, maxHeight, pageSize) + if err != nil { + return err + } + ad.txids = newTxids + ad.complete = complete + ad.maxHeight = maxHeight + if complete { + ad.txs = uint32(len(ad.txids)) + if ad.txs != ad.balance.Txs { + glog.Warning("xpubCheckAndLoadTxids inconsistency ", ad.addrDesc, ", ad.txs=", ad.txs, ", ad.balance.Txs=", ad.balance.Txs) + } + } + return nil +} + +func (w *Worker) xpubDerivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, error) { var err error if ad.balance, err = w.db.GetAddrDescBalance(ad.addrDesc); err != nil { return false, err @@ -117,15 +160,19 @@ func (w *Worker) derivedAddressBalance(data *xpubData, ad *xpubAddress) (bool, e return false, nil } -func (w *Worker) scanAddresses(xpub string, data *xpubData, addresses []xpubAddress, gap int, change int, minDerivedIndex int, fork bool) (int, []xpubAddress, error) { +func (w *Worker) xpubScanAddresses(xpub string, data *xpubData, addresses []xpubAddress, gap int, change int, minDerivedIndex int, fork bool) (int, []xpubAddress, error) { // rescan known addresses lastUsed := 0 for i := range addresses { ad := &addresses[i] if fork { - ad.bottomHeight = 0 + // reset the cached data + ad.txs = 0 + ad.maxHeight = 0 + ad.complete = false + ad.txids = nil } - used, err := w.derivedAddressBalance(data, ad) + used, err := w.xpubDerivedAddressBalance(data, ad) if err != nil { return 0, nil, err } @@ -147,7 +194,7 @@ func (w *Worker) scanAddresses(xpub string, data *xpubData, addresses []xpubAddr } for i, a := range descriptors { ad := xpubAddress{addrDesc: a} - used, err := w.derivedAddressBalance(data, &ad) + used, err := w.xpubDerivedAddressBalance(data, &ad) if err != nil { return 0, nil, err } @@ -192,9 +239,19 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubsMux.Lock() data, found := cachedXpubs[xpub] cachedXpubsMux.Unlock() - // to load all data for xpub may take some time, perform it in a loop to process a possible new block + var ( + txm []string + txs []*Tx + txids []string + pg Paging + totalResults int + err error + bestheight uint32 + besthash string + ) + // to load all data for xpub may take some time, do it in a loop to process a possible new block for { - bestheight, besthash, err := w.db.GetBestBlock() + bestheight, besthash, err = w.db.GetBestBlock() if err != nil { return nil, errors.Annotatef(err, "GetBestBlock") } @@ -210,9 +267,8 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option return nil, err } if hash != data.dataHash { - // in case of for reset all cached txids + // in case of for reset all cached data fork = true - data.txids = nil } } processedHash = besthash @@ -220,19 +276,101 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option data.dataHeight = bestheight data.dataHash = besthash var lastUsedIndex int - lastUsedIndex, data.addresses, err = w.scanAddresses(xpub, data, data.addresses, gap, 0, 0, fork) + lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, data, data.addresses, gap, 0, 0, fork) if err != nil { return nil, err } - _, data.changeAddresses, err = w.scanAddresses(xpub, data, data.changeAddresses, gap, 1, lastUsedIndex, fork) + _, data.changeAddresses, err = w.xpubScanAddresses(xpub, data, data.changeAddresses, gap, 1, lastUsedIndex, fork) if err != nil { return nil, err } } + if option >= TxidHistory { + for i := range data.addresses { + if err = w.xpubCheckAndLoadTxids(&data.addresses[i], filter, bestheight, txsOnPage); err != nil { + return nil, err + } + } + for i := range data.changeAddresses { + if err = w.xpubCheckAndLoadTxids(&data.changeAddresses[i], filter, bestheight, txsOnPage); err != nil { + return nil, err + } + } + } } cachedXpubsMux.Lock() cachedXpubs[xpub] = data cachedXpubsMux.Unlock() + // TODO mempool + if option >= TxidHistory { + txc := make(xpubTxids, 0, 32) + var addTxids func(ad *xpubAddress) + if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { + addTxids = func(ad *xpubAddress) { + txc = append(txc, ad.txids...) + } + totalResults = int(data.txs) + } else { + toHeight := maxUint32 + if filter.ToHeight != 0 { + toHeight = filter.ToHeight + } + addTxids = func(ad *xpubAddress) { + for _, txid := range ad.txids { + if txid.height < filter.FromHeight || txid.height > toHeight { + continue + } + if filter.Vout != AddressFilterVoutOff { + if filter.Vout == AddressFilterVoutInputs && txid.inputOutput&txInput == 0 || + filter.Vout == AddressFilterVoutOutputs && txid.inputOutput&txOutput == 0 { + continue + } + } + txc = append(txc, txid) + } + } + totalResults = -1 + } + for i := range data.addresses { + addTxids(&data.addresses[i]) + } + for i := range data.changeAddresses { + addTxids(&data.changeAddresses[i]) + } + sort.Stable(txc) + var from, to int + pg, from, to, page = computePaging(len(txc), page, txsOnPage) + if len(txc) >= txsOnPage { + if totalResults < 0 { + pg.TotalPages = -1 + } else { + pg, _, _, _ = computePaging(totalResults, page, txsOnPage) + } + } + if option == TxidHistory { + txids = make([]string, len(txm)+to-from) + } else { + txs = make([]*Tx, len(txm)+to-from) + } + txi := 0 + // get confirmed transactions + for i := from; i < to; i++ { + xpubTxid := &txc[i] + if option == TxidHistory { + txids[txi] = xpubTxid.txid + } else { + if txs[txi], err = w.txFromTxid(xpubTxid.txid, bestheight, option); err != nil { + return nil, err + } + } + txi++ + } + if option == TxidHistory { + txids = txids[:txi] + } else if option >= TxHistoryLight { + txs = txs[:txi] + } + } totalTokens := 0 tokens := make([]Token, 0, 4) for i := range data.addresses { @@ -256,7 +394,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option var totalReceived big.Int totalReceived.Add(&data.balanceSat, &data.sentSat) addr := Address{ - // Paging: pg, + Paging: pg, AddrStr: xpub, BalanceSat: (*Amount)(&data.balanceSat), TotalReceivedSat: (*Amount)(&totalReceived), @@ -264,13 +402,11 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option Txs: int(data.txs), // UnconfirmedBalanceSat: (*Amount)(&uBalSat), // UnconfirmedTxs: len(txm), - // Transactions: txs, - // Txids: txids, - TotalTokens: totalTokens, - Tokens: tokens, - // Erc20Contract: erc20c, - // Nonce: nonce, + Transactions: txs, + Txids: txids, + TotalTokens: totalTokens, + Tokens: tokens, } - glog.Info("GetAddressForXpub ", xpub[:10], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) + glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) return &addr, nil } diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 059e32a3..54cb612d 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -26,12 +26,12 @@ {{$addr.Txs}} - Total XPUB addresses + Total XPUB Addresses {{$addr.TotalTokens}} {{- if $addr.Tokens -}} - {{if $data.AllTokens}}XPUB Addresses{{else}}Nonzero XPUB Addresses{{end}} + {{if $data.AllTokens}}XPUB Addresses{{else}}XPUB Addresses with Balance{{end}} @@ -93,12 +93,6 @@ - {{- if $addr.Tokens -}} - - {{- range $t := $addr.Tokens -}} - - {{- end -}} - {{- end -}}
From 266b0575b6ab140fa3e41f96055d9e1c73e7a02c Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 4 Feb 2019 16:53:49 +0100 Subject: [PATCH 10/26] Highlight xpub addresses in explorer --- api/types.go | 4 +++- api/xpub.go | 34 +++++++++++++++++++++++++++------- server/public.go | 19 ++++++++++++++++++- server/public_test.go | 12 ++++++------ server/socketio.go | 9 --------- static/css/main.css | 22 +++++++++++++++++++++- static/templates/txdetail.html | 22 +++++++++++----------- static/templates/xpub.html | 2 +- 8 files changed, 87 insertions(+), 37 deletions(-) diff --git a/api/types.go b/api/types.go index 6cea9243..86a48a03 100644 --- a/api/types.go +++ b/api/types.go @@ -233,7 +233,9 @@ type Address struct { TotalTokens int `json:"totalTokens,omitempty"` Tokens []Token `json:"tokens,omitempty"` Erc20Contract *bchain.Erc20Contract `json:"erc20contract,omitempty"` - Filter string `json:"-"` + // helpers for explorer + Filter string `json:"-"` + XPubAddresses map[string]struct{} `json:"-"` } // AddressUtxo holds information about address and its transactions diff --git a/api/xpub.go b/api/xpub.go index 141b8381..5a456003 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -235,6 +235,10 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option } // gap is increased one as there must be gap of empty addresses before the derivation is stopped gap++ + page-- + if page < 0 { + page = 0 + } var processedHash string cachedXpubsMux.Lock() data, found := cachedXpubs[xpub] @@ -305,7 +309,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if option >= TxidHistory { txc := make(xpubTxids, 0, 32) var addTxids func(ad *xpubAddress) - if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { + if filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff { addTxids = func(ad *xpubAddress) { txc = append(txc, ad.txids...) } @@ -372,13 +376,21 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option } } totalTokens := 0 + xpubAddresses := make(map[string]struct{}) tokens := make([]Token, 0, 4) for i := range data.addresses { ad := &data.addresses[i] if ad.balance != nil { totalTokens++ if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - tokens = append(tokens, w.tokenFromXpubAddress(ad, 0, i)) + t := w.tokenFromXpubAddress(ad, 0, i) + tokens = append(tokens, t) + xpubAddresses[t.Name] = struct{}{} + } else { + a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) + if len(a) > 0 { + xpubAddresses[a[0]] = struct{}{} + } } } } @@ -387,7 +399,14 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if ad.balance != nil { totalTokens++ if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - tokens = append(tokens, w.tokenFromXpubAddress(ad, 1, i)) + t := w.tokenFromXpubAddress(ad, 1, i) + tokens = append(tokens, t) + xpubAddresses[t.Name] = struct{}{} + } else { + a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) + if len(a) > 0 { + xpubAddresses[a[0]] = struct{}{} + } } } } @@ -402,10 +421,11 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option Txs: int(data.txs), // UnconfirmedBalanceSat: (*Amount)(&uBalSat), // UnconfirmedTxs: len(txm), - Transactions: txs, - Txids: txids, - TotalTokens: totalTokens, - Tokens: tokens, + Transactions: txs, + Txids: txids, + TotalTokens: totalTokens, + Tokens: tokens, + XPubAddresses: xpubAddresses, } glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) return &addr, nil diff --git a/server/public.go b/server/public.go index d40567c3..ecad94d0 100644 --- a/server/public.go +++ b/server/public.go @@ -415,7 +415,7 @@ func (s *PublicServer) parseTemplates() []*template.Template { "formatAmount": s.formatAmount, "formatAmountWithDecimals": formatAmountWithDecimals, "setTxToTemplateData": setTxToTemplateData, - "stringInSlice": stringInSlice, + "isOwnAddress": isOwnAddress, } var createTemplate func(filenames ...string) *template.Template if s.debug { @@ -504,6 +504,23 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { return td } +// returns true if addresses are "own", +// i.e. either the address of the address detail or belonging to the xpub +func isOwnAddress(td *TemplateData, addresses []string) bool { + if len(addresses) == 1 { + a := addresses[0] + if a == td.AddrStr { + return true + } + if td.Address != nil && td.Address.XPubAddresses != nil { + if _, found := td.Address.XPubAddresses[a]; found { + return true + } + } + } + return false +} + func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx var err error diff --git a/server/public_test.go b/server/public_test.go index 525a7248..a6feca10 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -151,7 +151,7 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { `td class="data">0 FAKE`, `mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj`, `13.60030331 FAKE`, - `
`, + ``, ``, }, }, @@ -168,8 +168,8 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { ``, `7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25`, `3172.83951061 FAKE ➡`, - ``, `mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj`, `13.60030331 FAKE`, - ``, + ``, ``, }, }, @@ -307,8 +307,8 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { ``, `7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25`, `3172.83951061 FAKE ➡`, - `
No Inputs (Newly Generated Coins)No Inputs (Newly Generated Coins)0.00012345 FAKEmtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, - `td>mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, + `mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, + `td>mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, `9172.83951061 FAKE ×`, `00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840`, ``, @@ -290,7 +290,7 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { `td class="data">0 FAKENo Inputs (Newly Generated Coins)No Inputs (Newly Generated Coins)0.00012345 FAKEmtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, - `td>mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, + `mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, + `td>mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL`, `9172.83951061 FAKE ×`, `00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840`, ``, diff --git a/server/socketio.go b/server/socketio.go index e742f3cb..fc70510d 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -292,15 +292,6 @@ type resultGetAddressHistory struct { } `json:"result"` } -func stringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - func txToResTx(tx *api.Tx) resTx { inputs := make([]txInputs, len(tx.Vin)) for i := range tx.Vin { diff --git a/static/css/main.css b/static/css/main.css index c4ef6a48..e94e97a9 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -182,6 +182,26 @@ h3 { text-transform: uppercase; } +.tx-own { + background-color: #fff6e6; +} + +.tx-amt { + float: right!important; +} + +.tx-in .tx-own .tx-amt { + color: #dc3545!important; +} + +.tx-out .tx-own .tx-amt { + color: #28a745!important; +} + +.tx-addr { + float: left!important; +} + .ellipsis { overflow: hidden; text-overflow: ellipsis; @@ -283,5 +303,5 @@ table.data-table table.data-table th { } .key { - color: #333 ; + color: #333; } \ No newline at end of file diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 97121b13..57bde9c9 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -1,4 +1,4 @@ -{{define "txdetail"}}{{$cs := .CoinShortcut}}{{$addr := .AddrStr}}{{$tx := .Tx}} +{{define "txdetail"}}{{$cs := .CoinShortcut}}{{$addr := .AddrStr}}{{$tx := .Tx}}{{$data := .}}
@@ -10,23 +10,23 @@
-
+
{{- range $vin := $tx.Vin -}} - + @@ -45,20 +45,20 @@
-
+
{{- if $vin.Txid -}} ➡  {{- end -}} {{- range $a := $vin.Addresses -}} - + {{if and (ne $a $addr) $vin.Searchable}}{{$a}}{{else}}{{$a}}{{end}} {{- else -}} - {{- if $vin.Hex -}}Unparsed address{{- else -}}No Inputs (Newly Generated Coins){{- end -}} + {{- if $vin.Hex -}}Unparsed address{{- else -}}No Inputs (Newly Generated Coins){{- end -}} {{- end -}}{{- if $vin.Addresses -}} - {{formatAmount $vin.ValueSat}} {{$cs}} + {{formatAmount $vin.ValueSat}} {{$cs}} {{- end -}}
{{- range $vout := $tx.Vout -}} - + - {{- if $addr.Tokens -}} + {{- if $addr.TotalTokens -}} - + {{- range $t := $addr.Tokens -}} - + {{- end -}} {{- if not $data.AllTokens -}} From ae332547baec9b4f13d94451240ea5baaa3c726e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 6 Feb 2019 17:35:10 +0100 Subject: [PATCH 12/26] Fix loading of txids for xpub --- api/xpub.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index a5528a32..b03b5700 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -126,6 +126,7 @@ func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, m } return err } + return nil } // unless the filter is completely off, load all txids if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { @@ -302,12 +303,12 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option } if option >= TxidHistory { for i := range data.addresses { - if err = w.xpubCheckAndLoadTxids(&data.addresses[i], filter, bestheight, txsOnPage); err != nil { + if err = w.xpubCheckAndLoadTxids(&data.addresses[i], filter, bestheight, (page+1)*txsOnPage); err != nil { return nil, err } } for i := range data.changeAddresses { - if err = w.xpubCheckAndLoadTxids(&data.changeAddresses[i], filter, bestheight, txsOnPage); err != nil { + if err = w.xpubCheckAndLoadTxids(&data.changeAddresses[i], filter, bestheight, (page+1)*txsOnPage); err != nil { return nil, err } } From 1ca4a0cfc79661e1c8cf6e83838c82e136816f5f Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 7 Feb 2019 17:28:45 +0100 Subject: [PATCH 13/26] Process mempool txs in api.GetAddress even for Basic level of details --- api/worker.go | 151 ++++++++++++++++++++++++-------------------------- 1 file changed, 72 insertions(+), 79 deletions(-) diff --git a/api/worker.go b/api/worker.go index d014db09..7b088560 100644 --- a/api/worker.go +++ b/api/worker.go @@ -615,6 +615,22 @@ func (w *Worker) txFromTxid(txid string, bestheight uint32, option GetAddressOpt return tx, nil } +func (w *Worker) getAddrDescAndNormalizeAddress(address string) (bchain.AddressDescriptor, string, error) { + addrDesc, err := w.chainParser.GetAddrDescFromAddress(address) + if err != nil { + return nil, "", NewAPIError(fmt.Sprintf("Invalid address, %v", err), true) + } + // convert the address to the format defined by the parser + addresses, _, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + if err != nil { + glog.V(2).Infof("GetAddressesFromAddrDesc error %v, %v", err, addrDesc) + } + if len(addresses) == 1 { + address = addresses[0] + } + return addrDesc, address, nil +} + // GetAddress computes address value and gets transactions for given address func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { start := time.Now() @@ -622,10 +638,6 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA if page < 0 { page = 0 } - addrDesc, err := w.chainParser.GetAddrDescFromAddress(address) - if err != nil { - return nil, NewAPIError(fmt.Sprintf("Invalid address, %v", err), true) - } var ( ba *db.AddrBalance tokens []Token @@ -640,6 +652,10 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA nonTokenTxs int totalResults int ) + addrDesc, address, err := w.getAddrDescAndNormalizeAddress(address) + if err != nil { + return nil, err + } if w.chainType == bchain.ChainEthereumType { var n uint64 ba, tokens, erc20c, n, nonTokenTxs, totalResults, err = w.getEthereumTypeAddressBalances(addrDesc, option, filter) @@ -662,90 +678,67 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA } } } - // get tx history if requested by option or check mempool if there are some transactions for a new address - if option >= TxidHistory || ba == nil { - // convert the address to the format defined by the parser - addresses, _, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + // if there are only unconfirmed transactions, there is no paging + if ba == nil { + ba = &db.AddrBalance{} + page = 0 + } + // process mempool, only if blockheight filter is off + if filter.FromHeight == 0 && filter.ToHeight == 0 { + txm, err = w.getAddressTxids(addrDesc, true, filter, maxInt) if err != nil { - glog.V(2).Infof("GetAddressesFromAddrDesc error %v, %v", err, addrDesc) + return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) } - if len(addresses) == 1 { - address = addresses[0] - } - // get txs from mempool only if blockheight filter is off - if filter.FromHeight == 0 && filter.ToHeight == 0 { - txm, err = w.getAddressTxids(addrDesc, true, filter, maxInt) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) - } - } - // if there are only unconfirmed transactions, there is no paging - if ba == nil { - ba = &db.AddrBalance{} - page = 0 - } - if option >= TxidHistory { - txc, err := w.getAddressTxids(addrDesc, false, filter, (page+1)*txsOnPage) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc) - } - bestheight, _, err := w.db.GetBestBlock() - if err != nil { - return nil, errors.Annotatef(err, "GetBestBlock") - } - var from, to int - pg, from, to, page = computePaging(len(txc), page, txsOnPage) - if len(txc) >= txsOnPage { - if totalResults < 0 { - pg.TotalPages = -1 - } else { - pg, _, _, _ = computePaging(totalResults, page, txsOnPage) - } - } - if option == TxidHistory { - txids = make([]string, len(txm)+to-from) + for _, txid := range txm { + tx, err := w.GetTransaction(txid, false, false) + // mempool transaction may fail + if err != nil || tx == nil { + glog.Warning("GetTransaction in mempool: ", err) } else { - txs = make([]*Tx, len(txm)+to-from) - } - txi := 0 - // get mempool transactions - for _, txid := range txm { - tx, err := w.GetTransaction(txid, false, false) - // mempool transaction may fail - if err != nil || tx == nil { - glog.Warning("GetTransaction in mempool: ", err) - } else { - // skip already confirmed txs, mempool may be out of sync - if tx.Confirmations == 0 { - uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) - uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) - if page == 0 { - if option == TxidHistory { - txids[txi] = tx.Txid - } else { - txs[txi] = tx - } - txi++ + // skip already confirmed txs, mempool may be out of sync + if tx.Confirmations == 0 { + uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) + uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) + if page == 0 { + if option == TxidHistory { + txids = append(txids, tx.Txid) + } else if option >= TxHistoryLight { + txs = append(txs, tx) } } } } - // get confirmed transactions - for i := from; i < to; i++ { - txid := txc[i] - if option == TxidHistory { - txids[txi] = txid - } else { - if txs[txi], err = w.txFromTxid(txid, bestheight, option); err != nil { - return nil, err - } - } - txi++ + } + } + // get tx history if requested by option or check mempool if there are some transactions for a new address + if option >= TxidHistory { + txc, err := w.getAddressTxids(addrDesc, false, filter, (page+1)*txsOnPage) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc) + } + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + var from, to int + pg, from, to, page = computePaging(len(txc), page, txsOnPage) + if len(txc) >= txsOnPage { + if totalResults < 0 { + pg.TotalPages = -1 + } else { + pg, _, _, _ = computePaging(totalResults, page, txsOnPage) } + } + for i := from; i < to; i++ { + txid := txc[i] if option == TxidHistory { - txids = txids[:txi] - } else if option >= TxHistoryLight { - txs = txs[:txi] + txids = append(txids, txid) + } else { + tx, err := w.txFromTxid(txid, bestheight, option) + if err != nil { + return nil, err + } + txs = append(txs, tx) } } } From 273436f10990e02e1e9f9442ea3abe29390eacef Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 8 Feb 2019 00:42:38 +0100 Subject: [PATCH 14/26] Include mempool transactions in GetAddressForXpub --- api/xpub.go | 205 +++++++++++++++++++++++++------------------- static/css/main.css | 2 +- 2 files changed, 116 insertions(+), 91 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index b03b5700..bc15da54 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -107,7 +107,7 @@ func (w *Worker) xpubGetAddressTxids(addrDesc bchain.AddressDescriptor, mempool return txs, complete, nil } -func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, maxHeight uint32, pageSize int) error { +func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, maxHeight uint32, maxResults int) error { // skip if not discovered if ad.balance == nil { return nil @@ -130,9 +130,9 @@ func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, m } // unless the filter is completely off, load all txids if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { - pageSize = maxInt + maxResults = maxInt } - newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, 0, maxHeight, pageSize) + newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, 0, maxHeight, maxResults) if err != nil { return err } @@ -245,9 +245,13 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubsMux.Lock() data, found := cachedXpubs[xpub] cachedXpubsMux.Unlock() + type mempoolMap struct { + tx *Tx + inputOutput byte + } var ( txc xpubTxids - txm []string + txmMap map[string]*Tx txs []*Tx txids []string pg Paging @@ -255,6 +259,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option err error bestheight uint32 besthash string + uBalSat big.Int ) // to load all data for xpub may take some time, do it in a loop to process a possible new block for { @@ -302,14 +307,11 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option glog.Info("Scanned ", len(data.addresses)+len(data.changeAddresses), " addresses in ", time.Since(start)) } if option >= TxidHistory { - for i := range data.addresses { - if err = w.xpubCheckAndLoadTxids(&data.addresses[i], filter, bestheight, (page+1)*txsOnPage); err != nil { - return nil, err - } - } - for i := range data.changeAddresses { - if err = w.xpubCheckAndLoadTxids(&data.changeAddresses[i], filter, bestheight, (page+1)*txsOnPage); err != nil { - return nil, err + for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { + for i := range da { + if err = w.xpubCheckAndLoadTxids(&da[i], filter, bestheight, (page+1)*txsOnPage); err != nil { + return nil, err + } } } } @@ -317,41 +319,85 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubsMux.Lock() cachedXpubs[xpub] = data cachedXpubsMux.Unlock() - // TODO mempool - if option >= TxidHistory { - txc = make(xpubTxids, 0, 32) - var addTxids func(ad *xpubAddress) - if filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff { - addTxids = func(ad *xpubAddress) { - txc = append(txc, ad.txids...) + // setup filtering of txids + var useTxids func(txid *xpubTxid, ad *xpubAddress) bool + var addTxids func(ad *xpubAddress) + if filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff { + addTxids = func(ad *xpubAddress) { + txc = append(txc, ad.txids...) + } + totalResults = int(data.txs) + } else { + toHeight := maxUint32 + if filter.ToHeight != 0 { + toHeight = filter.ToHeight + } + useTxids = func(txid *xpubTxid, ad *xpubAddress) bool { + if txid.height < filter.FromHeight || txid.height > toHeight { + return false } - totalResults = int(data.txs) - } else { - toHeight := maxUint32 - if filter.ToHeight != 0 { - toHeight = filter.ToHeight + if filter.Vout != AddressFilterVoutOff { + if filter.Vout == AddressFilterVoutInputs && txid.inputOutput&txInput == 0 || + filter.Vout == AddressFilterVoutOutputs && txid.inputOutput&txOutput == 0 { + return false + } } - addTxids = func(ad *xpubAddress) { - for _, txid := range ad.txids { - if txid.height < filter.FromHeight || txid.height > toHeight { - continue - } - if filter.Vout != AddressFilterVoutOff { - if filter.Vout == AddressFilterVoutInputs && txid.inputOutput&txInput == 0 || - filter.Vout == AddressFilterVoutOutputs && txid.inputOutput&txOutput == 0 { - continue - } - } + return true + } + addTxids = func(ad *xpubAddress) { + for _, txid := range ad.txids { + if useTxids(&txid, ad) { txc = append(txc, txid) } } - totalResults = -1 } - for i := range data.addresses { - addTxids(&data.addresses[i]) + totalResults = -1 + } + // process mempool, only if blockheight filter is off + if filter.FromHeight == 0 && filter.ToHeight == 0 { + txmMap = make(map[string]*Tx) + for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { + for i := range da { + ad := &da[i] + newTxids, _, err := w.xpubGetAddressTxids(ad.addrDesc, true, 0, 0, maxInt) + if err != nil { + return nil, err + } + for _, txid := range newTxids { + // the same tx can have multiple addresses from the same xpub, get it from backend it only once + tx, foundTx := txmMap[txid.txid] + if !foundTx { + tx, err = w.GetTransaction(txid.txid, false, false) + // mempool transaction may fail + if err != nil || tx == nil { + glog.Warning("GetTransaction in mempool: ", err) + continue + } + txmMap[txid.txid] = tx + } + // skip already confirmed txs, mempool may be out of sync + if tx.Confirmations == 0 { + uBalSat.Add(&uBalSat, tx.getAddrVoutValue(ad.addrDesc)) + uBalSat.Sub(&uBalSat, tx.getAddrVinValue(ad.addrDesc)) + if page == 0 && !foundTx && (useTxids == nil || useTxids(&txid, ad)) { + if option == TxidHistory { + txids = append(txids, tx.Txid) + } else if option >= TxHistoryLight { + txs = append(txs, tx) + } + } + } + + } + } } - for i := range data.changeAddresses { - addTxids(&data.changeAddresses[i]) + } + if option >= TxidHistory { + txc = make(xpubTxids, 0, 32) + for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { + for i := range da { + addTxids(&da[i]) + } } sort.Stable(txc) var from, to int @@ -363,58 +409,37 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option pg, _, _, _ = computePaging(totalResults, page, txsOnPage) } } - if option == TxidHistory { - txids = make([]string, len(txm)+to-from) - } else { - txs = make([]*Tx, len(txm)+to-from) - } - txi := 0 // get confirmed transactions for i := from; i < to; i++ { xpubTxid := &txc[i] if option == TxidHistory { - txids[txi] = xpubTxid.txid + txids = append(txids, xpubTxid.txid) } else { - if txs[txi], err = w.txFromTxid(xpubTxid.txid, bestheight, option); err != nil { + tx, err := w.txFromTxid(xpubTxid.txid, bestheight, option) + if err != nil { return nil, err } + txs = append(txs, tx) } - txi++ - } - if option == TxidHistory { - txids = txids[:txi] - } else if option >= TxHistoryLight { - txs = txs[:txi] } } totalTokens := 0 xpubAddresses := make(map[string]struct{}) tokens := make([]Token, 0, 4) - for i := range data.addresses { - ad := &data.addresses[i] - if ad.balance != nil { - totalTokens++ - if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - t := w.tokenFromXpubAddress(data, ad, 0, i) - tokens = append(tokens, t) - xpubAddresses[t.Name] = struct{}{} - } else { - a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) - if len(a) > 0 { - xpubAddresses[a[0]] = struct{}{} + for ci, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { + for i := range da { + ad := &da[i] + var t *Token + if ad.balance != nil { + totalTokens++ + if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { + token := w.tokenFromXpubAddress(data, ad, ci, i) + tokens = append(tokens, token) + t = &token + xpubAddresses[t.Name] = struct{}{} } } - } - } - for i := range data.changeAddresses { - ad := &data.changeAddresses[i] - if ad.balance != nil { - totalTokens++ - if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - t := w.tokenFromXpubAddress(data, ad, 1, i) - tokens = append(tokens, t) - xpubAddresses[t.Name] = struct{}{} - } else { + if t == nil { a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) if len(a) > 0 { xpubAddresses[a[0]] = struct{}{} @@ -425,19 +450,19 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option var totalReceived big.Int totalReceived.Add(&data.balanceSat, &data.sentSat) addr := Address{ - Paging: pg, - AddrStr: xpub, - BalanceSat: (*Amount)(&data.balanceSat), - TotalReceivedSat: (*Amount)(&totalReceived), - TotalSentSat: (*Amount)(&data.sentSat), - Txs: int(data.txs), - // UnconfirmedBalanceSat: (*Amount)(&uBalSat), - // UnconfirmedTxs: len(txm), - Transactions: txs, - Txids: txids, - TotalTokens: totalTokens, - Tokens: tokens, - XPubAddresses: xpubAddresses, + Paging: pg, + AddrStr: xpub, + BalanceSat: (*Amount)(&data.balanceSat), + TotalReceivedSat: (*Amount)(&totalReceived), + TotalSentSat: (*Amount)(&data.sentSat), + Txs: int(data.txs), + UnconfirmedBalanceSat: (*Amount)(&uBalSat), + UnconfirmedTxs: len(txmMap), + Transactions: txs, + Txids: txids, + TotalTokens: totalTokens, + Tokens: tokens, + XPubAddresses: xpubAddresses, } glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs, loaded ", len(txc), " txids, finished in ", time.Since(start)) return &addr, nil diff --git a/static/css/main.css b/static/css/main.css index e94e97a9..d54f9841 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -183,7 +183,7 @@ h3 { } .tx-own { - background-color: #fff6e6; + background-color: #fbf8f0; } .tx-amt { From 6b0a4960fd9aea5ed57c1385c0dae99121de1c47 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 8 Feb 2019 15:50:37 +0100 Subject: [PATCH 15/26] Get utxo for xpub --- api/types.go | 29 +++++- api/typesv1.go | 2 +- api/worker.go | 238 +++++++++++++++++++++++++---------------------- api/xpub.go | 146 +++++++++++++++++++---------- server/public.go | 27 ++++-- 5 files changed, 268 insertions(+), 174 deletions(-) diff --git a/api/types.go b/api/types.go index 86a48a03..2efd46ae 100644 --- a/api/types.go +++ b/api/types.go @@ -213,7 +213,10 @@ type AddressFilter struct { Contract string FromHeight uint32 ToHeight uint32 - AllTokens bool + // AllTokens set to true will include xpub addresses with zero balance + AllTokens bool + // OnlyConfirmed set to true will ignore mempool transactions; mempool is also ignored if FromHeight/ToHeight filter is specified + OnlyConfirmed bool } // Address holds information about address and its transactions @@ -238,13 +241,33 @@ type Address struct { XPubAddresses map[string]struct{} `json:"-"` } -// AddressUtxo holds information about address and its transactions -type AddressUtxo struct { +// Utxo is one unspent transaction output +type Utxo struct { Txid string `json:"txid"` Vout int32 `json:"vout"` AmountSat *Amount `json:"value"` Height int `json:"height,omitempty"` Confirmations int `json:"confirmations"` + Address string `json:"address,omitempty"` + Path string `json:"path,omitempty"` +} + +// Utxos is array of Utxo +type Utxos []Utxo + +func (a Utxos) Len() int { return len(a) } +func (a Utxos) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a Utxos) Less(i, j int) bool { + // sort in reverse order, unconfirmed (height==0) utxos on top + hi := a[i].Height + hj := a[j].Height + if hi == 0 { + hi = maxInt + } + if hj == 0 { + hj = maxInt + } + return hi >= hj } // Blocks is list of blocks with paging information diff --git a/api/typesv1.go b/api/typesv1.go index 52703d49..bcdad73f 100644 --- a/api/typesv1.go +++ b/api/typesv1.go @@ -192,7 +192,7 @@ func (w *Worker) AddressToV1(a *Address) *AddressV1 { } // AddressUtxoToV1 converts []AddressUtxo to []AddressUtxoV1 -func (w *Worker) AddressUtxoToV1(au []AddressUtxo) []AddressUtxoV1 { +func (w *Worker) AddressUtxoToV1(au Utxos) []AddressUtxoV1 { d := w.chainParser.AmountDecimals() v1 := make([]AddressUtxoV1, len(au)) for i := range au { diff --git a/api/worker.go b/api/worker.go index 7b088560..5e084f7e 100644 --- a/api/worker.go +++ b/api/worker.go @@ -684,7 +684,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA page = 0 } // process mempool, only if blockheight filter is off - if filter.FromHeight == 0 && filter.ToHeight == 0 { + if filter.FromHeight == 0 && filter.ToHeight == 0 && !filter.OnlyConfirmed { txm, err = w.getAddressTxids(addrDesc, true, filter, maxInt) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) @@ -766,126 +766,138 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA return r, nil } +func (w *Worker) getAddrDescUtxo(addrDesc bchain.AddressDescriptor, ba *db.AddrBalance, onlyConfirmed bool, onlyMempool bool) (Utxos, error) { + var err error + r := make(Utxos, 0, 8) + spentInMempool := make(map[string]struct{}) + if !onlyConfirmed { + // get utxo from mempool + txm, err := w.getAddressTxids(addrDesc, true, &AddressFilter{Vout: AddressFilterVoutOff}, maxInt) + if err != nil { + return nil, err + } + if len(txm) > 0 { + mc := make([]*bchain.Tx, len(txm)) + for i, txid := range txm { + // get mempool txs and process their inputs to detect spends between mempool txs + bchainTx, _, err := w.txCache.GetTransaction(txid) + // mempool transaction may fail + if err != nil { + glog.Error("GetTransaction in mempool ", txid, ": ", err) + } else { + mc[i] = bchainTx + // get outputs spent by the mempool tx + for i := range bchainTx.Vin { + vin := &bchainTx.Vin[i] + spentInMempool[vin.Txid+strconv.Itoa(int(vin.Vout))] = struct{}{} + } + } + } + for _, bchainTx := range mc { + if bchainTx != nil { + for i := range bchainTx.Vout { + vout := &bchainTx.Vout[i] + vad, err := w.chainParser.GetAddrDescFromVout(vout) + if err == nil && bytes.Equal(addrDesc, vad) { + // report only outpoints that are not spent in mempool + _, e := spentInMempool[bchainTx.Txid+strconv.Itoa(i)] + if !e { + r = append(r, Utxo{ + Txid: bchainTx.Txid, + Vout: int32(i), + AmountSat: (*Amount)(&vout.ValueSat), + }) + } + } + } + } + } + } + } + if !onlyMempool { + // get utxo from index + if ba == nil { + ba, err = w.db.GetAddrDescBalance(addrDesc) + if err != nil { + return nil, NewAPIError(fmt.Sprintf("Address not found, %v", err), true) + } + } + // ba can be nil if the address is only in mempool! + if ba != nil && !IsZeroBigInt(&ba.BalanceSat) { + outpoints := make([]bchain.Outpoint, 0, 8) + err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, height uint32, indexes []int32) error { + for _, index := range indexes { + // take only outputs + if index >= 0 { + outpoints = append(outpoints, bchain.Outpoint{Txid: txid, Vout: index}) + } + } + return nil + }) + if err != nil { + return nil, err + } + var lastTxid string + var ta *db.TxAddresses + var checksum big.Int + checksum.Set(&ba.BalanceSat) + b, _, err := w.db.GetBestBlock() + if err != nil { + return nil, err + } + bestheight := int(b) + for i := len(outpoints) - 1; i >= 0 && checksum.Int64() > 0; i-- { + o := outpoints[i] + if lastTxid != o.Txid { + ta, err = w.db.GetTxAddresses(o.Txid) + if err != nil { + return nil, err + } + lastTxid = o.Txid + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", o.Txid, ": not found in txAddresses") + } else { + if len(ta.Outputs) <= int(o.Vout) { + glog.Warning("DB inconsistency: txAddresses ", o.Txid, " does not have enough outputs") + } else { + if !ta.Outputs[o.Vout].Spent { + v := ta.Outputs[o.Vout].ValueSat + // report only outpoints that are not spent in mempool + _, e := spentInMempool[o.Txid+strconv.Itoa(int(o.Vout))] + if !e { + r = append(r, Utxo{ + Txid: o.Txid, + Vout: o.Vout, + AmountSat: (*Amount)(&v), + Height: int(ta.Height), + Confirmations: bestheight - int(ta.Height) + 1, + }) + } + checksum.Sub(&checksum, &v) + } + } + } + } + if checksum.Uint64() != 0 { + glog.Warning("DB inconsistency: ", addrDesc, ": checksum is not zero") + } + } + } + return r, nil +} + // GetAddressUtxo returns unspent outputs for given address -func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUtxo, error) { +func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) (Utxos, error) { if w.chainType != bchain.ChainBitcoinType { return nil, NewAPIError("Not supported", true) } start := time.Now() addrDesc, err := w.chainParser.GetAddrDescFromAddress(address) if err != nil { - return nil, NewAPIError(fmt.Sprintf("Invalid address, %v", err), true) - } - spentInMempool := make(map[string]struct{}) - r := make([]AddressUtxo, 0, 8) - if !onlyConfirmed { - // get utxo from mempool - txm, err := w.getAddressTxids(addrDesc, true, &AddressFilter{Vout: AddressFilterVoutOff}, maxInt) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v true", address) - } - mc := make([]*bchain.Tx, len(txm)) - for i, txid := range txm { - // get mempool txs and process their inputs to detect spends between mempool txs - bchainTx, _, err := w.txCache.GetTransaction(txid) - // mempool transaction may fail - if err != nil { - glog.Error("GetTransaction in mempool ", txid, ": ", err) - } else { - mc[i] = bchainTx - // get outputs spent by the mempool tx - for i := range bchainTx.Vin { - vin := &bchainTx.Vin[i] - spentInMempool[vin.Txid+strconv.Itoa(int(vin.Vout))] = struct{}{} - } - } - } - for _, bchainTx := range mc { - if bchainTx != nil { - for i := range bchainTx.Vout { - vout := &bchainTx.Vout[i] - vad, err := w.chainParser.GetAddrDescFromVout(vout) - if err == nil && bytes.Equal(addrDesc, vad) { - // report only outpoints that are not spent in mempool - _, e := spentInMempool[bchainTx.Txid+strconv.Itoa(i)] - if !e { - r = append(r, AddressUtxo{ - Txid: bchainTx.Txid, - Vout: int32(i), - AmountSat: (*Amount)(&vout.ValueSat), - }) - } - } - } - } - } - } - // get utxo from index - ba, err := w.db.GetAddrDescBalance(addrDesc) - if err != nil { - return nil, NewAPIError(fmt.Sprintf("Address not found, %v", err), true) - } - var checksum big.Int - // ba can be nil if the address is only in mempool! - if ba != nil && !IsZeroBigInt(&ba.BalanceSat) { - outpoints := make([]bchain.Outpoint, 0, 8) - err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, height uint32, indexes []int32) error { - for _, index := range indexes { - // take only outputs - if index >= 0 { - outpoints = append(outpoints, bchain.Outpoint{Txid: txid, Vout: index}) - } - } - return nil - }) - if err != nil { - return nil, err - } - var lastTxid string - var ta *db.TxAddresses - checksum = ba.BalanceSat - b, _, err := w.db.GetBestBlock() - if err != nil { - return nil, err - } - bestheight := int(b) - for i := len(outpoints) - 1; i >= 0 && checksum.Int64() > 0; i-- { - o := outpoints[i] - if lastTxid != o.Txid { - ta, err = w.db.GetTxAddresses(o.Txid) - if err != nil { - return nil, err - } - lastTxid = o.Txid - } - if ta == nil { - glog.Warning("DB inconsistency: tx ", o.Txid, ": not found in txAddresses") - } else { - if len(ta.Outputs) <= int(o.Vout) { - glog.Warning("DB inconsistency: txAddresses ", o.Txid, " does not have enough outputs") - } else { - if !ta.Outputs[o.Vout].Spent { - v := ta.Outputs[o.Vout].ValueSat - // report only outpoints that are not spent in mempool - _, e := spentInMempool[o.Txid+strconv.Itoa(int(o.Vout))] - if !e { - r = append(r, AddressUtxo{ - Txid: o.Txid, - Vout: o.Vout, - AmountSat: (*Amount)(&v), - Height: int(ta.Height), - Confirmations: bestheight - int(ta.Height) + 1, - }) - } - checksum.Sub(&checksum, &v) - } - } - } - } - } - if checksum.Uint64() != 0 { - glog.Warning("DB inconsistency: ", address, ": checksum is not zero") + return nil, NewAPIError(fmt.Sprintf("Invalid address '%v', %v", address, err), true) } + r, err := w.getAddrDescUtxo(addrDesc, nil, onlyConfirmed, false) glog.Info("GetAddressUtxo ", address, ", ", len(r), " utxos, finished in ", time.Since(start)) return r, nil } diff --git a/api/xpub.go b/api/xpub.go index bc15da54..6fa86932 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -19,7 +19,7 @@ const defaultAddressesGap = 20 const txInput = 1 const txOutput = 2 -var cachedXpubs = make(map[string]*xpubData) +var cachedXpubs = make(map[string]xpubData) var cachedXpubsMux sync.Mutex type xpubTxid struct { @@ -216,63 +216,52 @@ func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeInd if len(a) > 0 { address = a[0] } + var balance *big.Int + var transfers int + if ad.balance != nil { + balance = &ad.balance.BalanceSat + transfers = int(ad.balance.Txs) + } return Token{ Type: XPUBAddressTokenType, Name: address, Decimals: w.chainParser.AmountDecimals(), - BalanceSat: (*Amount)(&ad.balance.BalanceSat), - Transfers: int(ad.balance.Txs), + BalanceSat: (*Amount)(balance), + Transfers: transfers, Contract: fmt.Sprintf("%s/%d/%d", data.basePath, changeIndex, index), } } -// GetAddressForXpub computes address value and gets transactions for given address -func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*Address, error) { +func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*xpubData, uint32, error) { if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { - return nil, ErrUnsupportedXpub + return nil, 0, ErrUnsupportedXpub } - start := time.Now() + var ( + err error + bestheight uint32 + besthash string + ) if gap <= 0 { gap = defaultAddressesGap } // gap is increased one as there must be gap of empty addresses before the derivation is stopped gap++ - page-- - if page < 0 { - page = 0 - } var processedHash string cachedXpubsMux.Lock() data, found := cachedXpubs[xpub] cachedXpubsMux.Unlock() - type mempoolMap struct { - tx *Tx - inputOutput byte - } - var ( - txc xpubTxids - txmMap map[string]*Tx - txs []*Tx - txids []string - pg Paging - totalResults int - err error - bestheight uint32 - besthash string - uBalSat big.Int - ) // to load all data for xpub may take some time, do it in a loop to process a possible new block for { bestheight, besthash, err = w.db.GetBestBlock() if err != nil { - return nil, errors.Annotatef(err, "GetBestBlock") + return nil, 0, errors.Annotatef(err, "GetBestBlock") } if besthash == processedHash { break } fork := false if !found || data.gap != gap { - data = &xpubData{gap: gap} + data = xpubData{gap: gap} data.basePath, err = w.chainParser.DerivationBasePath(xpub) if err != nil { glog.Warning("DerivationBasePath error", err) @@ -281,7 +270,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option } else { hash, err := w.db.GetBlockHash(data.dataHeight) if err != nil { - return nil, err + return nil, 0, err } if hash != data.dataHash { // in case of for reset all cached data @@ -296,21 +285,20 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option data.sentSat = *new(big.Int) data.txs = 0 var lastUsedIndex int - lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, data, data.addresses, gap, 0, 0, fork) + lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, &data, data.addresses, gap, 0, 0, fork) if err != nil { - return nil, err + return nil, 0, err } - _, data.changeAddresses, err = w.xpubScanAddresses(xpub, data, data.changeAddresses, gap, 1, lastUsedIndex, fork) + _, data.changeAddresses, err = w.xpubScanAddresses(xpub, &data, data.changeAddresses, gap, 1, lastUsedIndex, fork) if err != nil { - return nil, err + return nil, 0, err } - glog.Info("Scanned ", len(data.addresses)+len(data.changeAddresses), " addresses in ", time.Since(start)) } if option >= TxidHistory { for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { if err = w.xpubCheckAndLoadTxids(&da[i], filter, bestheight, (page+1)*txsOnPage); err != nil { - return nil, err + return nil, 0, err } } } @@ -319,6 +307,34 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubsMux.Lock() cachedXpubs[xpub] = data cachedXpubsMux.Unlock() + return &data, bestheight, nil +} + +// GetXpubAddress computes address value and gets transactions for given address +func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*Address, error) { + start := time.Now() + page-- + if page < 0 { + page = 0 + } + type mempoolMap struct { + tx *Tx + inputOutput byte + } + var ( + txc xpubTxids + txmMap map[string]*Tx + txs []*Tx + txids []string + pg Paging + totalResults int + err error + uBalSat big.Int + ) + data, bestheight, err := w.getXpubData(xpub, page, txsOnPage, option, filter, gap) + if err != nil { + return nil, err + } // setup filtering of txids var useTxids func(txid *xpubTxid, ad *xpubAddress) bool var addTxids func(ad *xpubAddress) @@ -354,7 +370,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option totalResults = -1 } // process mempool, only if blockheight filter is off - if filter.FromHeight == 0 && filter.ToHeight == 0 { + if filter.FromHeight == 0 && filter.ToHeight == 0 && !filter.OnlyConfirmed { txmMap = make(map[string]*Tx) for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { @@ -429,22 +445,14 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option for ci, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { ad := &da[i] - var t *Token + token := w.tokenFromXpubAddress(data, ad, ci, i) if ad.balance != nil { totalTokens++ if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - token := w.tokenFromXpubAddress(data, ad, ci, i) tokens = append(tokens, token) - t = &token - xpubAddresses[t.Name] = struct{}{} - } - } - if t == nil { - a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) - if len(a) > 0 { - xpubAddresses[a[0]] = struct{}{} } } + xpubAddresses[token.Name] = struct{}{} } } var totalReceived big.Int @@ -464,6 +472,48 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option Tokens: tokens, XPubAddresses: xpubAddresses, } - glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs, loaded ", len(txc), " txids, finished in ", time.Since(start)) + glog.Info("GetXpubAddress ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs, loaded ", len(txc), " txids, finished in ", time.Since(start)) return &addr, nil } + +// GetXpubUtxo returns unspent outputs for given xpub +func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, error) { + start := time.Now() + data, _, err := w.getXpubData(xpub, 0, 1, Basic, &AddressFilter{ + Vout: AddressFilterVoutOff, + AllTokens: false, + OnlyConfirmed: onlyConfirmed, + }, gap) + if err != nil { + return nil, err + } + r := make(Utxos, 0, 8) + for ci, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { + for i := range da { + ad := &da[i] + onlyMempool := false + if ad.balance == nil { + if onlyConfirmed { + continue + } + onlyMempool = true + } + utxos, err := w.getAddrDescUtxo(ad.addrDesc, ad.balance, onlyConfirmed, onlyMempool) + if err != nil { + return nil, err + } + if len(utxos) > 0 { + t := w.tokenFromXpubAddress(data, ad, ci, i) + for j := range utxos { + a := &utxos[j] + a.Address = t.Name + a.Path = t.Contract + } + r = append(r, utxos...) + } + } + } + sort.Stable(r) + glog.Info("GetXpubUtxo ", xpub[:16], ", ", len(r), " utxos, finished in ", time.Since(start)) + return r, nil +} diff --git a/server/public.go b/server/public.go index ecad94d0..e1ffec45 100644 --- a/server/public.go +++ b/server/public.go @@ -157,7 +157,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/v1/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV1)) serveMux.HandleFunc(path+"api/v1/tx/", s.jsonHandler(s.apiTx, apiV1)) serveMux.HandleFunc(path+"api/v1/address/", s.jsonHandler(s.apiAddress, apiV1)) - serveMux.HandleFunc(path+"api/v1/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV1)) + serveMux.HandleFunc(path+"api/v1/utxo/", s.jsonHandler(s.apiUtxo, apiV1)) serveMux.HandleFunc(path+"api/v1/block/", s.jsonHandler(s.apiBlock, apiV1)) serveMux.HandleFunc(path+"api/v1/sendtx/", s.jsonHandler(s.apiSendTx, apiV1)) serveMux.HandleFunc(path+"api/v1/estimatefee/", s.jsonHandler(s.apiEstimateFee, apiV1)) @@ -167,7 +167,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx, apiDefault)) serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress, apiDefault)) serveMux.HandleFunc(path+"api/xpub/", s.jsonHandler(s.apiXpub, apiDefault)) - serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo, apiDefault)) + serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiUtxo, apiDefault)) serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock, apiDefault)) serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx, apiDefault)) serveMux.HandleFunc(path+"api/estimatefee/", s.jsonHandler(s.apiEstimateFee, apiDefault)) @@ -177,7 +177,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"api/v2/tx/", s.jsonHandler(s.apiTx, apiV2)) serveMux.HandleFunc(path+"api/v2/address/", s.jsonHandler(s.apiAddress, apiV2)) serveMux.HandleFunc(path+"api/v2/xpub/", s.jsonHandler(s.apiXpub, apiV2)) - serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV2)) + serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiUtxo, apiV2)) serveMux.HandleFunc(path+"api/v2/block/", s.jsonHandler(s.apiBlock, apiV2)) serveMux.HandleFunc(path+"api/v2/sendtx/", s.jsonHandler(s.apiSendTx, apiV2)) serveMux.HandleFunc(path+"api/v2/estimatefee/", s.jsonHandler(s.apiEstimateFee, apiV2)) @@ -625,7 +625,7 @@ func (s *PublicServer) getAddressForXpub(r *http.Request, xpub string, pageSize gap = 0 } allAddresses, _ := strconv.ParseBool(r.URL.Query().Get("alladdresses")) - return s.api.GetAddressForXpub(xpub, page, pageSize, option, &api.AddressFilter{Vout: fn, AllTokens: allAddresses}, gap) + return s.api.GetXpubAddress(xpub, page, pageSize, option, &api.AddressFilter{Vout: fn, AllTokens: allAddresses}, gap) } func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { @@ -714,7 +714,7 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t var err error s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() if len(q) > 0 { - address, err = s.api.GetAddressForXpub(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, 0) + address, err = s.api.GetXpubAddress(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, 0) if err == nil { http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302) return noTpl, nil, nil @@ -912,10 +912,9 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er return address, err } -func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interface{}, error) { - var utxo []api.AddressUtxo +func (s *PublicServer) apiUtxo(r *http.Request, apiVersion int) (interface{}, error) { + var utxo []api.Utxo var err error - s.metrics.ExplorerViews.With(common.Labels{"action": "api-address"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { onlyConfirmed := false c := r.URL.Query().Get("confirmed") @@ -925,7 +924,17 @@ func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interfac return nil, api.NewAPIError("Parameter 'confirmed' cannot be converted to boolean", true) } } - utxo, err = s.api.GetAddressUtxo(r.URL.Path[i+1:], onlyConfirmed) + gap, ec := strconv.Atoi(r.URL.Query().Get("gap")) + if ec != nil { + gap = 0 + } + utxo, err = s.api.GetXpubUtxo(r.URL.Path[i+1:], onlyConfirmed, gap) + if err == nil { + s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub-utxo"}).Inc() + } else { + utxo, err = s.api.GetAddressUtxo(r.URL.Path[i+1:], onlyConfirmed) + s.metrics.ExplorerViews.With(common.Labels{"action": "api-address-utxo"}).Inc() + } if err == nil && apiVersion == apiV1 { return s.api.AddressUtxoToV1(utxo), nil } From 63fb910ecb8727c82b0e1a75a206d23bc78420b5 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 11 Feb 2019 11:54:34 +0100 Subject: [PATCH 16/26] Add xpub handling to websocket interface --- api/xpub.go | 6 ++++-- server/public.go | 6 +++--- server/websocket.go | 20 +++++++++++++++----- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index 6fa86932..4ae2d9d0 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -112,7 +112,7 @@ func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, m if ad.balance == nil { return nil } - // if completely read, check if there are not some new txs and load if necessary + // if completely loaded, check if there are not some new txs and load if necessary if ad.complete { if ad.balance.Txs != ad.txs { newTxids, _, err := w.xpubGetAddressTxids(ad.addrDesc, false, ad.maxHeight+1, maxHeight, maxInt) @@ -129,10 +129,12 @@ func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, m return nil } // unless the filter is completely off, load all txids + // could be optimized to reflect filter.FromHeight, filter.ToHeight but this way it is simple and robust + fromHeight := uint32(0) if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { maxResults = maxInt } - newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, 0, maxHeight, maxResults) + newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, fromHeight, maxHeight, maxResults) if err != nil { return err } diff --git a/server/public.go b/server/public.go index e1ffec45..674e9820 100644 --- a/server/public.go +++ b/server/public.go @@ -600,7 +600,7 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( return addressTpl, data, nil } -func (s *PublicServer) getAddressForXpub(r *http.Request, xpub string, pageSize int, option api.GetAddressOption) (*api.Address, error) { +func (s *PublicServer) getXpubAddress(r *http.Request, xpub string, pageSize int, option api.GetAddressOption) (*api.Address, error) { var fn = api.AddressFilterVoutOff page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { @@ -633,7 +633,7 @@ func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl var err error s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - address, err = s.getAddressForXpub(r, r.URL.Path[i+1:], txsOnPage, api.TxHistoryLight) + address, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsOnPage, api.TxHistoryLight) if err != nil { return errorTpl, nil, err } @@ -904,7 +904,7 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - address, err = s.getAddressForXpub(r, r.URL.Path[i+1:], txsInAPI, api.TxidHistory) + address, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsInAPI, api.TxidHistory) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } diff --git a/server/websocket.go b/server/websocket.go index 91bfbab8..5377910b 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -317,10 +317,10 @@ func (s *WebsocketServer) onRequest(c *websocketChannel, req *websocketReq) { } if err == nil { glog.V(1).Info("Client ", c.id, " onRequest ", req.Method, " success") - s.metrics.SocketIORequests.With(common.Labels{"method": req.Method, "status": "success"}).Inc() + s.metrics.WebsocketRequests.With(common.Labels{"method": req.Method, "status": "success"}).Inc() } else { glog.Error("Client ", c.id, " onMessage ", req.Method, ": ", errors.ErrorStack(err)) - s.metrics.SocketIORequests.With(common.Labels{"method": req.Method, "status": err.Error()}).Inc() + s.metrics.WebsocketRequests.With(common.Labels{"method": req.Method, "status": err.Error()}).Inc() e := resultError{} e.Error.Message = err.Error() data = e @@ -358,16 +358,26 @@ func (s *WebsocketServer) getAccountInfo(req *accountInfoReq) (res *api.Address, default: opt = api.Basic } - return s.api.GetAddress(req.Descriptor, req.Page, req.PageSize, opt, &api.AddressFilter{ + filter := api.AddressFilter{ FromHeight: uint32(req.FromHeight), ToHeight: uint32(req.ToHeight), Contract: req.ContractFilter, Vout: api.AddressFilterVoutOff, - }) + AllTokens: true, + } + a, err := s.api.GetXpubAddress(req.Descriptor, req.Page, req.PageSize, opt, &filter, 0) + if err != nil { + return s.api.GetAddress(req.Descriptor, req.Page, req.PageSize, opt, &filter) + } + return a, nil } func (s *WebsocketServer) getAccountUtxo(descriptor string) (interface{}, error) { - return s.api.GetAddressUtxo(descriptor, false) + utxo, err := s.api.GetXpubUtxo(descriptor, false, 0) + if err != nil { + return s.api.GetAddressUtxo(descriptor, false) + } + return utxo, nil } func (s *WebsocketServer) getTransaction(txid string) (interface{}, error) { From 4846af9f60d0d5f8126e82a1c43a799a2c112d85 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 12 Feb 2019 15:15:10 +0100 Subject: [PATCH 17/26] Control token detail level returned by xpub api --- api/types.go | 42 +++++++++++++++--------- api/worker.go | 12 +++---- api/xpub.go | 43 +++++++++++++++---------- server/public.go | 65 ++++++++++++++++++++++---------------- server/websocket.go | 6 ++-- static/templates/xpub.html | 8 ++--- static/test-websocket.html | 2 +- 7 files changed, 105 insertions(+), 73 deletions(-) diff --git a/api/types.go b/api/types.go index 2efd46ae..0d1f1b4f 100644 --- a/api/types.go +++ b/api/types.go @@ -19,13 +19,13 @@ type GetAddressOption int const ( // Basic - only that address is indexed and some basic info Basic GetAddressOption = iota - // Balance - only balances - Balance - // TxidHistory - balances and txids, subject to paging + // Tokens - basic info + tokens + Tokens + // TxidHistory - basic + tokens + txids, subject to paging TxidHistory - // TxHistoryLight - balances and easily obtained tx data (not requiring request to backend), subject to paging + // TxHistoryLight - basic + tokens + easily obtained tx data (not requiring request to backend), subject to paging TxHistoryLight - // TxHistory - balances and full tx data, subject to paging + // TxHistory - basic + tokens + full tx data, subject to paging TxHistory ) @@ -138,14 +138,17 @@ const XPUBAddressTokenType TokenType = "XPUBAddress" // Token contains info about tokens held by an address type Token struct { - Type TokenType `json:"type"` - Contract string `json:"contract"` - Transfers int `json:"transfers"` - Name string `json:"name"` - Symbol string `json:"symbol"` - Decimals int `json:"decimals"` - BalanceSat *Amount `json:"balance,omitempty"` - ContractIndex string `json:"-"` + Type TokenType `json:"type"` + Name string `json:"name"` + Path string `json:"path,omitempty"` + Contract string `json:"contract,omitempty"` + Transfers int `json:"transfers"` + Symbol string `json:"symbol,omitempty"` + Decimals int `json:"decimals,omitempty"` + BalanceSat *Amount `json:"balance,omitempty"` + TotalReceivedSat *Amount `json:"totalReceived,omitempty"` + TotalSentSat *Amount `json:"totalSent,omitempty"` + ContractIndex string `json:"-"` } // TokenTransfer contains info about a token transfer done in a transaction @@ -198,6 +201,9 @@ type Paging struct { ItemsOnPage int `json:"itemsOnPage,omitempty"` } +// TokenDetailLevel specifies detail level of tokens returned by GetAddress and GetXpubAddress +type TokenDetailLevel int + const ( // AddressFilterVoutOff disables filtering of transactions by vout AddressFilterVoutOff = -1 @@ -205,6 +211,13 @@ const ( AddressFilterVoutInputs = -2 // AddressFilterVoutOutputs specifies that only txs where the address is as output are returned AddressFilterVoutOutputs = -3 + + // TokenDetailNonzeroBalance - use to return only tokens with nonzero balance + TokenDetailNonzeroBalance TokenDetailLevel = 0 + // TokenDetailUsed - use to return tokens with some transfers (even if they have zero balance now) + TokenDetailUsed TokenDetailLevel = 1 + // TokenDetailDiscovered - use to return all discovered tokens + TokenDetailDiscovered TokenDetailLevel = 2 ) // AddressFilter is used to filter data returned from GetAddress api method @@ -213,8 +226,7 @@ type AddressFilter struct { Contract string FromHeight uint32 ToHeight uint32 - // AllTokens set to true will include xpub addresses with zero balance - AllTokens bool + TokenLevel TokenDetailLevel // OnlyConfirmed set to true will ignore mempool transactions; mempool is also ignored if FromHeight/ToHeight filter is specified OnlyConfirmed bool } diff --git a/api/worker.go b/api/worker.go index 5e084f7e..80e19871 100644 --- a/api/worker.go +++ b/api/worker.go @@ -51,7 +51,7 @@ func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescript // setSpendingTxToVout is helper function, that finds transaction that spent given output and sets it to the output // there is no direct index for the operation, it must be found using addresses -> txaddresses -> tx func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) error { - err := w.db.GetAddrDescTransactions(vout.AddrDesc, height, ^uint32(0), func(t string, height uint32, indexes []int32) error { + err := w.db.GetAddrDescTransactions(vout.AddrDesc, height, maxUint32, func(t string, height uint32, indexes []int32) error { for _, index := range indexes { // take only inputs if index < 0 { @@ -364,7 +364,7 @@ func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool } else { to := filter.ToHeight if to == 0 { - to = ^uint32(0) + to = maxUint32 } err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, callback) if err != nil { @@ -683,8 +683,8 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA ba = &db.AddrBalance{} page = 0 } - // process mempool, only if blockheight filter is off - if filter.FromHeight == 0 && filter.ToHeight == 0 && !filter.OnlyConfirmed { + // process mempool, only if toHeight is not specified + if filter.ToHeight == 0 && !filter.OnlyConfirmed { txm, err = w.getAddressTxids(addrDesc, true, filter, maxInt) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) @@ -825,7 +825,7 @@ func (w *Worker) getAddrDescUtxo(addrDesc bchain.AddressDescriptor, ba *db.AddrB // ba can be nil if the address is only in mempool! if ba != nil && !IsZeroBigInt(&ba.BalanceSat) { outpoints := make([]bchain.Outpoint, 0, 8) - err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, height uint32, indexes []int32) error { + err = w.db.GetAddrDescTransactions(addrDesc, 0, maxUint32, func(txid string, height uint32, indexes []int32) error { for _, index := range indexes { // take only outputs if index >= 0 { @@ -943,7 +943,7 @@ func (w *Worker) GetBlock(bid string, page int, txsOnPage int) (*Block, error) { // if it's a number, must be less than int32 var hash string height, err := strconv.Atoi(bid) - if err == nil && height < int(^uint32(0)) { + if err == nil && height < int(maxUint32) { hash, err = w.db.GetBlockHash(uint32(height)) if err != nil { hash = bid diff --git a/api/xpub.go b/api/xpub.go index 4ae2d9d0..c0076275 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -218,19 +218,23 @@ func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeInd if len(a) > 0 { address = a[0] } - var balance *big.Int + var balance, totalReceived, totalSent *big.Int var transfers int if ad.balance != nil { balance = &ad.balance.BalanceSat + totalSent = &ad.balance.SentSat + totalReceived = ad.balance.ReceivedSat() transfers = int(ad.balance.Txs) } return Token{ - Type: XPUBAddressTokenType, - Name: address, - Decimals: w.chainParser.AmountDecimals(), - BalanceSat: (*Amount)(balance), - Transfers: transfers, - Contract: fmt.Sprintf("%s/%d/%d", data.basePath, changeIndex, index), + Type: XPUBAddressTokenType, + Name: address, + Decimals: w.chainParser.AmountDecimals(), + BalanceSat: (*Amount)(balance), + TotalReceivedSat: (*Amount)(totalReceived), + TotalSentSat: (*Amount)(totalSent), + Transfers: transfers, + Path: fmt.Sprintf("%s/%d/%d", data.basePath, changeIndex, index), } } @@ -371,8 +375,8 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } totalResults = -1 } - // process mempool, only if blockheight filter is off - if filter.FromHeight == 0 && filter.ToHeight == 0 && !filter.OnlyConfirmed { + // process mempool, only if ToHeight is not specified + if filter.ToHeight == 0 && !filter.OnlyConfirmed { txmMap = make(map[string]*Tx) for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { @@ -442,19 +446,27 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } } totalTokens := 0 - xpubAddresses := make(map[string]struct{}) - tokens := make([]Token, 0, 4) + var tokens []Token + var xpubAddresses map[string]struct{} + if option != Basic { + tokens = make([]Token, 0, 4) + xpubAddresses = make(map[string]struct{}) + } for ci, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { ad := &da[i] - token := w.tokenFromXpubAddress(data, ad, ci, i) if ad.balance != nil { totalTokens++ - if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { + } + if option != Basic { + token := w.tokenFromXpubAddress(data, ad, ci, i) + if filter.TokenLevel == TokenDetailDiscovered || + filter.TokenLevel == TokenDetailUsed && ad.balance != nil || + filter.TokenLevel == TokenDetailNonzeroBalance && ad.balance != nil && !IsZeroBigInt(&ad.balance.BalanceSat) { tokens = append(tokens, token) } + xpubAddresses[token.Name] = struct{}{} } - xpubAddresses[token.Name] = struct{}{} } } var totalReceived big.Int @@ -483,7 +495,6 @@ func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, e start := time.Now() data, _, err := w.getXpubData(xpub, 0, 1, Basic, &AddressFilter{ Vout: AddressFilterVoutOff, - AllTokens: false, OnlyConfirmed: onlyConfirmed, }, gap) if err != nil { @@ -509,7 +520,7 @@ func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, e for j := range utxos { a := &utxos[j] a.Address = t.Name - a.Path = t.Contract + a.Path = t.Path } r = append(r, utxos...) } diff --git a/server/public.go b/server/public.go index 674e9820..7b507c02 100644 --- a/server/public.go +++ b/server/public.go @@ -385,27 +385,27 @@ const ( // TemplateData is used to transfer data to the templates type TemplateData struct { - CoinName string - CoinShortcut string - CoinLabel string - InternalExplorer bool - ChainType bchain.ChainType - Address *api.Address - AddrStr string - Tx *api.Tx - Error *api.APIError - Blocks *api.Blocks - Block *api.Block - Info *api.SystemInfo - Page int - PrevPage int - NextPage int - PagingRange []int - PageParams template.URL - TOSLink string - SendTxHex string - Status string - AllTokens bool + CoinName string + CoinShortcut string + CoinLabel string + InternalExplorer bool + ChainType bchain.ChainType + Address *api.Address + AddrStr string + Tx *api.Tx + Error *api.APIError + Blocks *api.Blocks + Block *api.Block + Info *api.SystemInfo + Page int + PrevPage int + NextPage int + PagingRange []int + PageParams template.URL + TOSLink string + SendTxHex string + Status string + NonZeroBalanceTokens bool } func (s *PublicServer) parseTemplates() []*template.Template { @@ -600,7 +600,7 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( return addressTpl, data, nil } -func (s *PublicServer) getXpubAddress(r *http.Request, xpub string, pageSize int, option api.GetAddressOption) (*api.Address, error) { +func (s *PublicServer) getXpubAddress(r *http.Request, xpub string, pageSize int, option api.GetAddressOption) (*api.Address, api.TokenDetailLevel, error) { var fn = api.AddressFilterVoutOff page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { @@ -624,16 +624,26 @@ func (s *PublicServer) getXpubAddress(r *http.Request, xpub string, pageSize int if ec != nil { gap = 0 } - allAddresses, _ := strconv.ParseBool(r.URL.Query().Get("alladdresses")) - return s.api.GetXpubAddress(xpub, page, pageSize, option, &api.AddressFilter{Vout: fn, AllTokens: allAddresses}, gap) + tokenLevel := api.TokenDetailNonzeroBalance + switch r.URL.Query().Get("tokenlevel") { + case "discovered": + tokenLevel = api.TokenDetailDiscovered + case "used": + tokenLevel = api.TokenDetailUsed + case "nonzero": + tokenLevel = api.TokenDetailNonzeroBalance + } + a, err := s.api.GetXpubAddress(xpub, page, pageSize, option, &api.AddressFilter{Vout: fn, TokenLevel: tokenLevel}, gap) + return a, tokenLevel, err } func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var address *api.Address + var tokenLevel api.TokenDetailLevel var err error s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - address, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsOnPage, api.TxHistoryLight) + address, tokenLevel, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsOnPage, api.TxHistoryLight) if err != nil { return errorTpl, nil, err } @@ -648,8 +658,7 @@ func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl data.PageParams = template.URL("&filter=" + filter) data.Address.Filter = filter } - allAddresses := r.URL.Query().Get("alladdresses") - data.AllTokens, _ = strconv.ParseBool(allAddresses) + data.NonZeroBalanceTokens = tokenLevel == api.TokenDetailNonzeroBalance return xpubTpl, data, nil } @@ -904,7 +913,7 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - address, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsInAPI, api.TxidHistory) + address, _, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsInAPI, api.TxidHistory) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } diff --git a/server/websocket.go b/server/websocket.go index 5377910b..aa1d746e 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -349,8 +349,8 @@ func unmarshalGetAccountInfoRequest(params []byte) (*accountInfoReq, error) { func (s *WebsocketServer) getAccountInfo(req *accountInfoReq) (res *api.Address, err error) { var opt api.GetAddressOption switch req.Details { - case "balance": - opt = api.Balance + case "tokens": + opt = api.Tokens case "txids": opt = api.TxidHistory case "txs": @@ -363,7 +363,7 @@ func (s *WebsocketServer) getAccountInfo(req *accountInfoReq) (res *api.Address, ToHeight: uint32(req.ToHeight), Contract: req.ContractFilter, Vout: api.AddressFilterVoutOff, - AllTokens: true, + TokenLevel: api.TokenDetailDiscovered, } a, err := s.api.GetXpubAddress(req.Descriptor, req.Page, req.PageSize, opt, &filter, 0) if err != nil { diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 743f5ff4..7ee8f429 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -31,7 +31,7 @@ {{- if $addr.TotalTokens -}} - +
{{- range $a := $vout.Addresses -}} - + {{- if and (ne $a $addr) $vout.Searchable}}{{$a}}{{else}}{{$a}}{{- end -}} {{- else -}} - Unparsed address + Unparsed address {{- end -}} - + {{formatAmount $vout.ValueSat}} {{$cs}} {{if $vout.Spent}}➡{{else -}} × {{- end -}} diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 54cb612d..1696c4a7 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -29,7 +29,7 @@ Total XPUB Addresses {{$addr.TotalTokens}}
{{if $data.AllTokens}}XPUB Addresses{{else}}XPUB Addresses with Balance{{end}} From 64c8ae9a62265c60e151533863e9f973a45d726b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 5 Feb 2019 20:47:54 +0100 Subject: [PATCH 11/26] Estimate full derivation path from xpub --- Gopkg.lock | 2 +- api/xpub.go | 25 +++++-- bchain/baseparser.go | 5 ++ bchain/coins/btc/bitcoinparser.go | 29 ++++++++ bchain/coins/btc/bitcoinparser_test.go | 69 ++++++++++++++++++++ bchain/coins/btc/bitcoinrpc.go | 1 + bchain/types.go | 1 + build/templates/blockbook/blockchaincfg.json | 1 + build/tools/templates.go | 1 + build/tools/trezor-common/sync-coins.go | 4 ++ configs/coins/bcash.json | 1 + configs/coins/bcash_testnet.json | 1 + configs/coins/bgold.json | 1 + configs/coins/bitcoin_testnet.json | 1 + configs/coins/dash.json | 1 + configs/coins/dash_testnet.json | 1 + configs/coins/digibyte.json | 1 + configs/coins/dogecoin.json | 1 + configs/coins/fujicoin.json | 1 + configs/coins/gamecredits.json | 1 + configs/coins/groestlcoin.json | 1 + configs/coins/groestlcoin_testnet.json | 1 + configs/coins/koto.json | 1 + configs/coins/litecoin.json | 1 + configs/coins/litecoin_testnet.json | 1 + configs/coins/monacoin.json | 1 + configs/coins/myriad.json | 1 + configs/coins/namecoin.json | 1 + configs/coins/vertcoin.json | 1 + configs/coins/zcash.json | 1 + configs/coins/zcash_testnet.json | 1 + configs/coins/zcoin.json | 1 + static/templates/xpub.html | 4 +- 33 files changed, 154 insertions(+), 10 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f1c9364c..4bbfcd31 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -95,7 +95,7 @@ branch = "master" name = "github.com/martinboehm/btcutil" packages = [".","base58","bech32","chaincfg","hdkeychain","txscript"] - revision = "520c2dbb6e0420531b6ae148845280378516e971" + revision = "63034958e64b209cb9294128309dbaed497cde7b" [[projects]] branch = "master" diff --git a/api/xpub.go b/api/xpub.go index 5a456003..a5528a32 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -45,6 +45,7 @@ type xpubAddress struct { type xpubData struct { gap int + basePath string dataHeight uint32 dataHash string txs uint32 @@ -208,7 +209,7 @@ func (w *Worker) xpubScanAddresses(xpub string, data *xpubData, addresses []xpub return lastUsed, addresses, nil } -func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index int) Token { +func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeIndex int, index int) Token { a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) var address string if len(a) > 0 { @@ -220,7 +221,7 @@ func (w *Worker) tokenFromXpubAddress(ad *xpubAddress, changeIndex int, index in Decimals: w.chainParser.AmountDecimals(), BalanceSat: (*Amount)(&ad.balance.BalanceSat), Transfers: int(ad.balance.Txs), - Contract: fmt.Sprintf("%d/%d", changeIndex, index), + Contract: fmt.Sprintf("%s/%d/%d", data.basePath, changeIndex, index), } } @@ -244,6 +245,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option data, found := cachedXpubs[xpub] cachedXpubsMux.Unlock() var ( + txc xpubTxids txm []string txs []*Tx txids []string @@ -265,6 +267,11 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option fork := false if !found || data.gap != gap { data = &xpubData{gap: gap} + data.basePath, err = w.chainParser.DerivationBasePath(xpub) + if err != nil { + glog.Warning("DerivationBasePath error", err) + data.basePath = "unknown" + } } else { hash, err := w.db.GetBlockHash(data.dataHeight) if err != nil { @@ -276,9 +283,12 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option } } processedHash = besthash - if data.dataHeight < bestheight { + if data.dataHeight < bestheight || fork { data.dataHeight = bestheight data.dataHash = besthash + data.balanceSat = *new(big.Int) + data.sentSat = *new(big.Int) + data.txs = 0 var lastUsedIndex int lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, data, data.addresses, gap, 0, 0, fork) if err != nil { @@ -288,6 +298,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if err != nil { return nil, err } + glog.Info("Scanned ", len(data.addresses)+len(data.changeAddresses), " addresses in ", time.Since(start)) } if option >= TxidHistory { for i := range data.addresses { @@ -307,7 +318,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option cachedXpubsMux.Unlock() // TODO mempool if option >= TxidHistory { - txc := make(xpubTxids, 0, 32) + txc = make(xpubTxids, 0, 32) var addTxids func(ad *xpubAddress) if filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff { addTxids = func(ad *xpubAddress) { @@ -383,7 +394,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if ad.balance != nil { totalTokens++ if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - t := w.tokenFromXpubAddress(ad, 0, i) + t := w.tokenFromXpubAddress(data, ad, 0, i) tokens = append(tokens, t) xpubAddresses[t.Name] = struct{}{} } else { @@ -399,7 +410,7 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option if ad.balance != nil { totalTokens++ if filter.AllTokens || !IsZeroBigInt(&ad.balance.BalanceSat) { - t := w.tokenFromXpubAddress(ad, 1, i) + t := w.tokenFromXpubAddress(data, ad, 1, i) tokens = append(tokens, t) xpubAddresses[t.Name] = struct{}{} } else { @@ -427,6 +438,6 @@ func (w *Worker) GetAddressForXpub(xpub string, page int, txsOnPage int, option Tokens: tokens, XPubAddresses: xpubAddresses, } - glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs finished in ", time.Since(start)) + glog.Info("GetAddressForXpub ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs, loaded ", len(txc), " txids, finished in ", time.Since(start)) return &addr, nil } diff --git a/bchain/baseparser.go b/bchain/baseparser.go index dbba33a5..cc8ec98e 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -267,6 +267,11 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { return &tx, pt.Height, nil } +// DerivationBasePath is unsupported +func (p *BaseParser) DerivationBasePath(xpub string) (string, error) { + return "", errors.New("Not supported") +} + // DeriveAddressDescriptors is unsupported func (p *BaseParser) DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]AddressDescriptor, error) { return nil, errors.New("Not supported") diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index d4f13b02..e6a71482 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/hex" "math/big" + "strconv" vlq "github.com/bsm/go-vlq" "github.com/juju/errors" @@ -28,6 +29,7 @@ type BitcoinParser struct { XPubMagic uint32 XPubMagicSegwitP2sh uint32 XPubMagicSegwitNative uint32 + Slip44 uint32 } // NewBitcoinParser returns new BitcoinParser instance @@ -41,6 +43,7 @@ func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser XPubMagic: c.XPubMagic, XPubMagicSegwitP2sh: c.XPubMagicSegwitP2sh, XPubMagicSegwitNative: c.XPubMagicSegwitNative, + Slip44: c.Slip44, } p.OutputScriptToAddressesFunc = p.outputScriptToAddresses return p @@ -349,3 +352,29 @@ func (p *BitcoinParser) DeriveAddressDescriptorsFromTo(xpub string, change uint3 } return ad, nil } + +// DerivationBasePath returns base path of xpub +func (p *BitcoinParser) DerivationBasePath(xpub string) (string, error) { + extKey, err := hdkeychain.NewKeyFromString(xpub) + if err != nil { + return "", err + } + var c, bip string + cn := extKey.ChildNum() + if cn >= 0x80000000 { + cn -= 0x80000000 + c = "'" + } + c = strconv.Itoa(int(cn)) + c + if extKey.Depth() != 3 { + return "unknown/" + c, nil + } + if extKey.Version() == p.XPubMagicSegwitP2sh { + bip = "49" + } else if extKey.Version() == p.XPubMagicSegwitNative { + bip = "84" + } else { + bip = "44" + } + return "m/" + bip + "'/" + strconv.Itoa(int(p.Slip44)) + "'/" + c, nil +} diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index f85aece1..deb15839 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -590,3 +590,72 @@ func BenchmarkDeriveAddressDescriptorsFromToZpub(b *testing.B) { btcMainParser.DeriveAddressDescriptorsFromTo("zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs", 1, 0, 100) } } + +func TestBitcoinParser_DerivationBasePath(t *testing.T) { + btcMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, XPubMagicSegwitP2sh: 77429938, XPubMagicSegwitNative: 78792518, Slip44: 0}) + btcTestnetsParser := NewBitcoinParser(GetChainParams("test"), &Configuration{XPubMagic: 70617039, XPubMagicSegwitP2sh: 71979618, XPubMagicSegwitNative: 73342198, Slip44: 1}) + zecMainParser := NewBitcoinParser(GetChainParams("main"), &Configuration{XPubMagic: 76067358, Slip44: 133}) + type args struct { + xpub string + parser *BitcoinParser + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "m/84'/0'/0'", + args: args{ + xpub: "zpub6rFR7y4Q2AijBEqTUquhVz398htDFrtymD9xYYfG1m4wAcvPhXNfE3EfH1r1ADqtfSdVCToUG868RvUUkgDKf31mGDtKsAYz2oz2AGutZYs", + parser: btcMainParser, + }, + want: "m/84'/0'/0'", + }, + { + name: "m/49'/0'/55 - not hardened account", + args: args{ + xpub: "ypub6XKbB5DJRAbW4TRJLp4uXQXG3ob5BtByXsNZFBjq9qcbzrczjVXfCz5cEo1SFDexmeWRnbCMDaRgaW4m9d2nBaa8FvUQCu3n9G1UBR8WhbT", + parser: btcMainParser, + }, + want: "m/49'/0'/55", + }, + { + name: "m/49'/0' - incomplete path, without account", + args: args{ + xpub: "ypub6UzM8PUqxcSoqC9gumfoiFhE8Qt84HbGpCD4eVJfJAojXTVtBxeddvTWJGJhGoaVBNJLmEgMdLXHgaLVJa4xEvk2tcokkdZhFdkxMLUE9sB", + parser: btcMainParser, + }, + want: "unknown/0'", + }, + { + name: "m/49'/1'/0'", + args: args{ + xpub: "upub5DR1Mg5nykixzYjFXWW5GghAU7dDqoPVJ2jrqFbL8sJ7Hs7jn69MP7KBnnmxn88GeZtnH8PRKV9w5MMSFX8AdEAoXY8Qd8BJPoXtpMeHMxJ", + parser: btcTestnetsParser, + }, + want: "m/49'/1'/0'", + }, + { + name: "m/44'/133'/12'", + args: args{ + xpub: "xpub6CQdEahwhKRTLYpP6cyb7ZaGb3r4tVdyPX6dC1PfrNuByrCkWDgUkmpD28UdV9QccKgY1ZiAbGv1Fakcg2LxdFVSTNKHcjdRjqhjPK8Trkb", + parser: zecMainParser, + }, + want: "m/44'/133'/12'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.parser.DerivationBasePath(tt.args.xpub) + if (err != nil) != tt.wantErr { + t.Errorf("BitcoinParser.DerivationBasePath() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("BitcoinParser.DerivationBasePath() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index e9f9e4ec..24e24ee8 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -54,6 +54,7 @@ type Configuration struct { XPubMagic uint32 `json:"xpub_magic,omitempty"` XPubMagicSegwitP2sh uint32 `json:"xpub_magic_segwit_p2sh,omitempty"` XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native,omitempty"` + Slip44 uint32 `json:"slip44,omitempty"` } // NewBitcoinRPC returns new BitcoinRPC instance. diff --git a/bchain/types.go b/bchain/types.go index f9e9245a..ead62535 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -264,6 +264,7 @@ type BlockChainParser interface { UnpackBlockHash(buf []byte) (string, error) ParseBlock(b []byte) (*Block, error) // xpub + DerivationBasePath(xpub string) (string, error) DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]AddressDescriptor, error) DeriveAddressDescriptorsFromTo(xpub string, change uint32, fromIndex uint32, toIndex uint32) ([]AddressDescriptor, error) // EthereumType specific diff --git a/build/templates/blockbook/blockchaincfg.json b/build/templates/blockbook/blockchaincfg.json index aea2ad96..525937c5 100644 --- a/build/templates/blockbook/blockchaincfg.json +++ b/build/templates/blockbook/blockchaincfg.json @@ -19,6 +19,7 @@ {{if .Blockbook.BlockChain.XPubMagic}} "xpub_magic": {{.Blockbook.BlockChain.XPubMagic}}, {{end}}{{if .Blockbook.BlockChain.XPubMagicSegwitP2sh}} "xpub_magic_segwit_p2sh": {{.Blockbook.BlockChain.XPubMagicSegwitP2sh}}, {{end}}{{if .Blockbook.BlockChain.XPubMagicSegwitNative}} "xpub_magic_segwit_native": {{.Blockbook.BlockChain.XPubMagicSegwitNative}}, +{{end}}{{if .Blockbook.BlockChain.Slip44}} "slip44": {{.Blockbook.BlockChain.Slip44}}, {{end}} "mempool_workers": {{.Blockbook.BlockChain.MempoolWorkers}}, "mempool_sub_workers": {{.Blockbook.BlockChain.MempoolSubWorkers}}, diff --git a/build/tools/templates.go b/build/tools/templates.go index 5194cd98..b8101ada 100644 --- a/build/tools/templates.go +++ b/build/tools/templates.go @@ -70,6 +70,7 @@ type Config struct { XPubMagic uint32 `json:"xpub_magic,omitempty"` XPubMagicSegwitP2sh uint32 `json:"xpub_magic_segwit_p2sh,omitempty"` XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native,omitempty"` + Slip44 uint32 `json:"slip44,omitempty"` AdditionalParams map[string]json.RawMessage `json:"additional_params"` } `json:"block_chain"` diff --git a/build/tools/trezor-common/sync-coins.go b/build/tools/trezor-common/sync-coins.go index a8aa4be0..7a522058 100644 --- a/build/tools/trezor-common/sync-coins.go +++ b/build/tools/trezor-common/sync-coins.go @@ -27,6 +27,7 @@ type trezorCommonDef struct { XPubMagic uint32 `json:"xpub_magic"` XPubMagicSegwitP2sh uint32 `json:"xpub_magic_segwit_p2sh"` XPubMagicSegwitNative uint32 `json:"xpub_magic_segwit_native"` + Slip44 uint32 `json:"slip44,omitempty"` } func getTrezorCommonDef(coin string) (*trezorCommonDef, error) { @@ -109,6 +110,9 @@ func main() { if tcd.XPubMagicSegwitNative != 0 { config.Blockbook.BlockChain.XPubMagicSegwitNative = tcd.XPubMagicSegwitNative } + if tcd.Slip44 != 0 { + config.Blockbook.BlockChain.Slip44 = tcd.Slip44 + } err = writeConfig(coin, config) if err == nil { fmt.Printf("%v updated\n", coin) diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 365e925f..27b0cbab 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -55,6 +55,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "slip44": 145, "additional_params": {} } }, diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index f8b65e7a..76687125 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -55,6 +55,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 70617039, + "slip44": 1, "additional_params": {} } }, diff --git a/configs/coins/bgold.json b/configs/coins/bgold.json index 063868b4..02633bd7 100644 --- a/configs/coins/bgold.json +++ b/configs/coins/bgold.json @@ -250,6 +250,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "slip44": 156, "additional_params": {} } }, diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index e1136f46..d137099c 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -58,6 +58,7 @@ "xpub_magic": 70617039, "xpub_magic_segwit_p2sh": 71979618, "xpub_magic_segwit_native": 73342198, + "slip44": 1, "additional_params": {} } }, diff --git a/configs/coins/dash.json b/configs/coins/dash.json index 7ddd1c58..4ce87346 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -57,6 +57,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 50221772, + "slip44": 5, "additional_params": {} } }, diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index 94b7cfae..7456994e 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -57,6 +57,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 70617039, + "slip44": 1, "additional_params": {} } }, diff --git a/configs/coins/digibyte.json b/configs/coins/digibyte.json index 08b4a47d..5deb852d 100644 --- a/configs/coins/digibyte.json +++ b/configs/coins/digibyte.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "slip44": 20, "additional_params": {} } }, diff --git a/configs/coins/dogecoin.json b/configs/coins/dogecoin.json index 1fcdc8df..43ae23cc 100644 --- a/configs/coins/dogecoin.json +++ b/configs/coins/dogecoin.json @@ -59,6 +59,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 49990397, + "slip44": 3, "additional_params": {} } }, diff --git a/configs/coins/fujicoin.json b/configs/coins/fujicoin.json index 0431b17d..11da3c99 100644 --- a/configs/coins/fujicoin.json +++ b/configs/coins/fujicoin.json @@ -54,6 +54,7 @@ "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, "xpub_magic_segwit_native": 78792518, + "slip44": 75, "additional_params": {} } }, diff --git a/configs/coins/gamecredits.json b/configs/coins/gamecredits.json index 7071bd82..dfb5efb5 100644 --- a/configs/coins/gamecredits.json +++ b/configs/coins/gamecredits.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 27106558, "xpub_magic_segwit_p2sh": 28471030, + "slip44": 101, "additional_params": {} } }, diff --git a/configs/coins/groestlcoin.json b/configs/coins/groestlcoin.json index 6ac5901a..e7abcd97 100644 --- a/configs/coins/groestlcoin.json +++ b/configs/coins/groestlcoin.json @@ -59,6 +59,7 @@ "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, "xpub_magic_segwit_native": 78792518, + "slip44": 17, "additional_params": {} } }, diff --git a/configs/coins/groestlcoin_testnet.json b/configs/coins/groestlcoin_testnet.json index 24f9d1af..2c3f3521 100644 --- a/configs/coins/groestlcoin_testnet.json +++ b/configs/coins/groestlcoin_testnet.json @@ -59,6 +59,7 @@ "xpub_magic": 70617039, "xpub_magic_segwit_p2sh": 71979618, "xpub_magic_segwit_native": 73342198, + "slip44": 1, "additional_params": {} } }, diff --git a/configs/coins/koto.json b/configs/coins/koto.json index 8c0e4fd4..63c19587 100644 --- a/configs/coins/koto.json +++ b/configs/coins/koto.json @@ -58,6 +58,7 @@ "mempool_sub_workers": 8, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "slip44": 510, "additional_params": {} } }, diff --git a/configs/coins/litecoin.json b/configs/coins/litecoin.json index 0ca821fb..600e8d34 100644 --- a/configs/coins/litecoin.json +++ b/configs/coins/litecoin.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 27108450, "xpub_magic_segwit_p2sh": 28471030, + "slip44": 2, "additional_params": {} } }, diff --git a/configs/coins/litecoin_testnet.json b/configs/coins/litecoin_testnet.json index 96dc2559..622bf912 100644 --- a/configs/coins/litecoin_testnet.json +++ b/configs/coins/litecoin_testnet.json @@ -56,6 +56,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 70617039, + "slip44": 1, "additional_params": {} } }, diff --git a/configs/coins/monacoin.json b/configs/coins/monacoin.json index d6c7b4d9..db281c81 100644 --- a/configs/coins/monacoin.json +++ b/configs/coins/monacoin.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "slip44": 22, "additional_params": {} } }, diff --git a/configs/coins/myriad.json b/configs/coins/myriad.json index 96f545f9..cbc2a903 100644 --- a/configs/coins/myriad.json +++ b/configs/coins/myriad.json @@ -56,6 +56,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "slip44": 90, "additional_params": {} } }, diff --git a/configs/coins/namecoin.json b/configs/coins/namecoin.json index 40b5a94e..bf964613 100644 --- a/configs/coins/namecoin.json +++ b/configs/coins/namecoin.json @@ -63,6 +63,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "slip44": 7, "additional_params": {} } }, diff --git a/configs/coins/vertcoin.json b/configs/coins/vertcoin.json index fab28ff2..0dfa7545 100644 --- a/configs/coins/vertcoin.json +++ b/configs/coins/vertcoin.json @@ -55,6 +55,7 @@ "block_addresses_to_keep": 1000, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "slip44": 28, "additional_params": {} } }, diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index 234f1982..f2455720 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -56,6 +56,7 @@ "mempool_sub_workers": 8, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "slip44": 133, "additional_params": {} } }, diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index 88096df4..50ba1b5b 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -56,6 +56,7 @@ "mempool_sub_workers": 8, "block_addresses_to_keep": 300, "xpub_magic": 70617039, + "slip44": 1, "additional_params": {} } }, diff --git a/configs/coins/zcoin.json b/configs/coins/zcoin.json index 6ac38c25..776b0207 100644 --- a/configs/coins/zcoin.json +++ b/configs/coins/zcoin.json @@ -67,6 +67,7 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "slip44": 136, "additional_params": {} } }, diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 1696c4a7..743f5ff4 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -39,14 +39,14 @@ Address Balance TxsPathPath
{{$t.Name}} {{formatAmount $t.BalanceSat}} {{$cs}} {{$t.Transfers}}{{$t.Contract}}{{$t.Contract}}
{{if $data.AllTokens}}XPUB Addresses{{else}}XPUB Addresses with Balance{{end}}{{if $data.NonZeroBalanceTokens}}XPUB Addresses with Balance{{else}}XPUB Addresses{{end}} @@ -46,12 +46,12 @@ - + {{- end -}} - {{- if not $data.AllTokens -}} + {{- if $data.NonZeroBalanceTokens -}} - + {{- end -}} diff --git a/static/test-websocket.html b/static/test-websocket.html index c5cca40d..ed28c05f 100644 --- a/static/test-websocket.html +++ b/static/test-websocket.html @@ -304,7 +304,7 @@ From f3ec1e6b77cfb73549abb20ace7d619d95fb171e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 13 Feb 2019 17:57:30 +0100 Subject: [PATCH 18/26] Return unique transactions from GetXpubAddress --- api/xpub.go | 71 ++++++++++++++++++-------------------- static/templates/xpub.html | 4 +-- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index c0076275..5a3a2052 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -48,7 +48,7 @@ type xpubData struct { basePath string dataHeight uint32 dataHash string - txs uint32 + txCountEstimate uint32 sentSat big.Int balanceSat big.Int addresses []xpubAddress @@ -108,7 +108,7 @@ func (w *Worker) xpubGetAddressTxids(addrDesc bchain.AddressDescriptor, mempool } func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, maxHeight uint32, maxResults int) error { - // skip if not discovered + // skip if not used if ad.balance == nil { return nil } @@ -128,13 +128,8 @@ func (w *Worker) xpubCheckAndLoadTxids(ad *xpubAddress, filter *AddressFilter, m } return nil } - // unless the filter is completely off, load all txids - // could be optimized to reflect filter.FromHeight, filter.ToHeight but this way it is simple and robust - fromHeight := uint32(0) - if filter.FromHeight != 0 || filter.ToHeight != 0 || filter.Vout != AddressFilterVoutOff { - maxResults = maxInt - } - newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, fromHeight, maxHeight, maxResults) + // load all txids to get paging correctly + newTxids, complete, err := w.xpubGetAddressTxids(ad.addrDesc, false, 0, maxHeight, maxInt) if err != nil { return err } @@ -156,7 +151,7 @@ func (w *Worker) xpubDerivedAddressBalance(data *xpubData, ad *xpubAddress) (boo return false, err } if ad.balance != nil { - data.txs += ad.balance.Txs + data.txCountEstimate += ad.balance.Txs data.sentSat.Add(&data.sentSat, &ad.balance.SentSat) data.balanceSat.Add(&data.balanceSat, &ad.balance.BalanceSat) return true, nil @@ -289,7 +284,7 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAdd data.dataHash = besthash data.balanceSat = *new(big.Int) data.sentSat = *new(big.Int) - data.txs = 0 + data.txCountEstimate = 0 var lastUsedIndex int lastUsedIndex, data.addresses, err = w.xpubScanAddresses(xpub, &data, data.addresses, gap, 0, 0, fork) if err != nil { @@ -328,14 +323,15 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get inputOutput byte } var ( - txc xpubTxids - txmMap map[string]*Tx - txs []*Tx - txids []string - pg Paging - totalResults int - err error - uBalSat big.Int + txc xpubTxids + txmMap map[string]*Tx + txcMap map[string]struct{} + txs []*Tx + txids []string + pg Paging + filtered bool + err error + uBalSat big.Int ) data, bestheight, err := w.getXpubData(xpub, page, txsOnPage, option, filter, gap) if err != nil { @@ -343,13 +339,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } // setup filtering of txids var useTxids func(txid *xpubTxid, ad *xpubAddress) bool - var addTxids func(ad *xpubAddress) - if filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff { - addTxids = func(ad *xpubAddress) { - txc = append(txc, ad.txids...) - } - totalResults = int(data.txs) - } else { + if !(filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff) { toHeight := maxUint32 if filter.ToHeight != 0 { toHeight = filter.ToHeight @@ -366,14 +356,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } return true } - addTxids = func(ad *xpubAddress) { - for _, txid := range ad.txids { - if useTxids(&txid, ad) { - txc = append(txc, txid) - } - } - } - totalResults = -1 + filtered = true } // process mempool, only if ToHeight is not specified if filter.ToHeight == 0 && !filter.OnlyConfirmed { @@ -414,14 +397,28 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } } } + txCount := int(data.txCountEstimate) if option >= TxidHistory { + txcMap = make(map[string]struct{}) txc = make(xpubTxids, 0, 32) for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { - addTxids(&da[i]) + ad := &da[i] + for _, txid := range ad.txids { + _, foundTx := txcMap[txid.txid] + if !foundTx && (useTxids == nil || useTxids(&txid, ad)) { + txcMap[txid.txid] = struct{}{} + txc = append(txc, txid) + } + } } } sort.Stable(txc) + txCount = len(txcMap) + totalResults := txCount + if filtered { + totalResults = -1 + } var from, to int pg, from, to, page = computePaging(len(txc), page, txsOnPage) if len(txc) >= txsOnPage { @@ -477,7 +474,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get BalanceSat: (*Amount)(&data.balanceSat), TotalReceivedSat: (*Amount)(&totalReceived), TotalSentSat: (*Amount)(&data.sentSat), - Txs: int(data.txs), + Txs: txCount, UnconfirmedBalanceSat: (*Amount)(&uBalSat), UnconfirmedTxs: len(txmMap), Transactions: txs, @@ -486,7 +483,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get Tokens: tokens, XPubAddresses: xpubAddresses, } - glog.Info("GetXpubAddress ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", data.txs, " total txs, loaded ", len(txc), " txids, finished in ", time.Since(start)) + glog.Info("GetXpubAddress ", xpub[:16], ", ", len(data.addresses)+len(data.changeAddresses), " derived addresses, ", txCount, " confirmed txs, finished in ", time.Since(start)) return &addr, nil } diff --git a/static/templates/xpub.html b/static/templates/xpub.html index 7ee8f429..a4b1f57f 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -26,10 +26,10 @@ - + - {{- if $addr.TotalTokens -}} + {{- if $addr.Tokens -}} `, + ``, + ``, + ``, + `
3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71
mined Tue, 21 Aug 2018 15:45:23 CEST
`, + `
effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75
mined Tue, 21 Aug 2018 15:27:01 CEST
`, + ``, + ``, + }, + }, { name: "explorerSearch not found", r: newGetRequest(ts.URL + "/search?q=1234"), @@ -437,7 +461,34 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { }, }, { - name: "apiAddressUtxo v1", + name: "apiXpub v2 tokenlevel=nonzero", + r: newGetRequest(ts.URL + "/api/v2/xpub/" + dbtestdata.Xpub), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"upub5E1xjDmZ7Hhej6LPpS8duATdKXnRYui7bDYj6ehfFGzWDZtmCmQkZhc3Zb7kgRLtHWd16QFxyP86JKL3ShZEBFX88aciJ3xyocuyhZZ8g6q","balance":"118641975500","totalReceived":"118641975501","totalSent":"1","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":2,"txids":["3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75"],"totalTokens":2,"tokens":[{"type":"XPUBAddress","name":"2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu","path":"m/49'/1'/33'/1/3","transfers":1,"decimals":8,"balance":"118641975500","totalReceived":"118641975500","totalSent":"0"}]}`, + }, + }, + { + name: "apiXpub v2 tokenlevel=used", + r: newGetRequest(ts.URL + "/api/v2/xpub/" + dbtestdata.Xpub + "?tokenlevel=used"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"upub5E1xjDmZ7Hhej6LPpS8duATdKXnRYui7bDYj6ehfFGzWDZtmCmQkZhc3Zb7kgRLtHWd16QFxyP86JKL3ShZEBFX88aciJ3xyocuyhZZ8g6q","balance":"118641975500","totalReceived":"118641975501","totalSent":"1","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":2,"txids":["3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75"],"totalTokens":2,"tokens":[{"type":"XPUBAddress","name":"2MzmAKayJmja784jyHvRUW1bXPget1csRRG","path":"m/49'/1'/33'/0/0","transfers":2,"decimals":8,"balance":"0","totalReceived":"1","totalSent":"1"},{"type":"XPUBAddress","name":"2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu","path":"m/49'/1'/33'/1/3","transfers":1,"decimals":8,"balance":"118641975500","totalReceived":"118641975500","totalSent":"0"}]}`, + }, + }, + { + name: "apiXpub v2 tokenlevel=discovered", + r: newGetRequest(ts.URL + "/api/v2/xpub/" + dbtestdata.Xpub + "?tokenlevel=discovered"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"upub5E1xjDmZ7Hhej6LPpS8duATdKXnRYui7bDYj6ehfFGzWDZtmCmQkZhc3Zb7kgRLtHWd16QFxyP86JKL3ShZEBFX88aciJ3xyocuyhZZ8g6q","balance":"118641975500","totalReceived":"118641975501","totalSent":"1","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":2,"txids":["3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75"],"totalTokens":2,"tokens":[{"type":"XPUBAddress","name":"2MzmAKayJmja784jyHvRUW1bXPget1csRRG","path":"m/49'/1'/33'/0/0","transfers":2,"decimals":8,"balance":"0","totalReceived":"1","totalSent":"1"},{"type":"XPUBAddress","name":"2MsYfbi6ZdVXLDNrYAQ11ja9Sd3otMk4Pmj","path":"m/49'/1'/33'/0/1","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MuAZNAjLSo6RLFad2fvHSfgqBD7BoEVy4T","path":"m/49'/1'/33'/0/2","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NEqKzw3BosGnBE9by5uaDy5QgwjHac4Zbg","path":"m/49'/1'/33'/0/3","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2Mw7vJNC8zUK6VNN4CEjtoTYmuNPLewxZzV","path":"m/49'/1'/33'/0/4","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N1kvo97NFASPXiwephZUxE9PRXunjTxEc4","path":"m/49'/1'/33'/0/5","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MuWrWMzoBt8VDFNvPmpJf42M1GTUs85fPx","path":"m/49'/1'/33'/0/6","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MuVZ2Ca6Da9zmYynt49Rx7uikAgubGcymF","path":"m/49'/1'/33'/0/7","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MzRGWDUmrPP9HwYu4B43QGCTLwoop5cExa","path":"m/49'/1'/33'/0/8","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N5C9EEWJzyBXhpyPHqa3UNed73Amsi5b3L","path":"m/49'/1'/33'/0/9","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MzNawz2zjwq1L85GDE3YydEJGJYfXxaWkk","path":"m/49'/1'/33'/0/10","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N7NdeuAMgL57WE7QCeV2gTWi2Um8iAu5dA","path":"m/49'/1'/33'/0/11","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N8JQEP6DSHEZHNsSDPA1gHMUq9YFndhkfV","path":"m/49'/1'/33'/0/12","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2Mvbn3YXqKZVpQKugaoQrfjSYPvz76RwZkC","path":"m/49'/1'/33'/0/13","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N8MRNxCfwUY9TSW27X9ooGYtqgrGCfLRHx","path":"m/49'/1'/33'/0/14","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N6HvwrHC113KYZAmCtJ9XJNWgaTcnFunCM","path":"m/49'/1'/33'/0/15","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NEo3oNyHUoi7rmRWee7wki37jxPWsWCopJ","path":"m/49'/1'/33'/0/16","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2Mzm5KY8qdFbDHsQfy4akXbFvbR3FAwDuVo","path":"m/49'/1'/33'/0/17","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NGMwftmQCogp6XZNGvgiybz3WZysvsJzqC","path":"m/49'/1'/33'/0/18","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N3fJrrefndYjLGycvFFfYgevpZtcRKCkRD","path":"m/49'/1'/33'/0/19","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N1T7TnHBwfdpBoyw53EGUL7vuJmb2mU6jF","path":"m/49'/1'/33'/0/20","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MzSBtRWHbBjeUcu3H5VRDqkvz5sfmDxJKo","path":"m/49'/1'/33'/1/0","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MtShtAJYb1afWduUTwF1SixJjan7urZKke","path":"m/49'/1'/33'/1/1","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N3cP668SeqyBEr9gnB4yQEmU3VyxeRYith","path":"m/49'/1'/33'/1/2","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu","path":"m/49'/1'/33'/1/3","transfers":1,"decimals":8,"balance":"118641975500","totalReceived":"118641975500","totalSent":"0"},{"type":"XPUBAddress","name":"2NEzatauNhf9kPTwwj6ZfYKjUdy52j4hVUL","path":"m/49'/1'/33'/1/4","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N4RjsDp4LBpkNqyF91aNjgpF9CwDwBkJZq","path":"m/49'/1'/33'/1/5","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N8XygTmQc4NoBBPEy3yybnfCYhsxFtzPDY","path":"m/49'/1'/33'/1/6","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N5BjBomZvb48sccK2vwLMiQ5ETKp1fdPVn","path":"m/49'/1'/33'/1/7","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2MybMwbZRPCGU3SMWPwQCpDkbcQFw5Hbwen","path":"m/49'/1'/33'/1/8","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N7HexL4dyAQc7Th4iqcCW4hZuyiZsLWf74","path":"m/49'/1'/33'/1/9","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NF6X5FDGWrQj4nQrfP6hA77zB5WAc1DGup","path":"m/49'/1'/33'/1/10","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N4ZRPdvc7BVioBTohy4F6QtxreqcjNj26b","path":"m/49'/1'/33'/1/11","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2Mtfho1rLmevh4qTnkYWxZEFCWteDMtTcUF","path":"m/49'/1'/33'/1/12","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NFUCphKYvmMcNZRZrF261mRX6iADVB9Qms","path":"m/49'/1'/33'/1/13","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N5kBNMB8qgxE4Y4f8J19fScsE49J4aNvoJ","path":"m/49'/1'/33'/1/14","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NANWCaefhCKdXMcW8NbZnnrFRDvhJN2wPy","path":"m/49'/1'/33'/1/15","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NFHw7Yo2Bz8D2wGAYHW9qidbZFLpfJ72qB","path":"m/49'/1'/33'/1/16","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NBDSsBgy5PpFniLCb1eAFHcSxgxwPSDsZa","path":"m/49'/1'/33'/1/17","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NDWCSQHogc7sCuc2WoYt9PX2i2i6a5k6dX","path":"m/49'/1'/33'/1/18","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N8vNyDP7iSDjm3BKpXrbDjAxyphqfvnJz8","path":"m/49'/1'/33'/1/19","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2N4tFKLurSbMusAyq1tv4tzymVjveAFV1Vb","path":"m/49'/1'/33'/1/20","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NBx5WwjAr2cH6Yqrp3Vsf957HtRKwDUVdX","path":"m/49'/1'/33'/1/21","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NBu1seHTaFhQxbcW5L5BkZzqFLGmZqpxsa","path":"m/49'/1'/33'/1/22","transfers":0,"decimals":8},{"type":"XPUBAddress","name":"2NCDLoea22jGsXuarfT1n2QyCUh6RFhAPnT","path":"m/49'/1'/33'/1/23","transfers":0,"decimals":8}]}`, + }, + }, + { + name: "apiUtxo v1", r: newGetRequest(ts.URL + "/api/v1/utxo/mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"), status: http.StatusOK, contentType: "application/json; charset=utf-8", @@ -446,7 +497,7 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { }, }, { - name: "apiAddressUtxo v2", + name: "apiUtxo v2", r: newGetRequest(ts.URL + "/api/v2/utxo/mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"), status: http.StatusOK, contentType: "application/json; charset=utf-8", @@ -454,6 +505,15 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { `[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vout":1,"value":"917283951061","height":225494,"confirmations":1}]`, }, }, + { + name: "apiUtxo v2 xpub", + r: newGetRequest(ts.URL + "/api/v2/utxo/" + dbtestdata.Xpub), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `[{"txid":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","vout":0,"value":"118641975500","height":225494,"confirmations":1,"address":"2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu","path":"m/49'/1'/33'/1/3"}]`, + }, + }, { name: "apiSendTx", r: newGetRequest(ts.URL + "/api/sendtx/1234567890"), @@ -599,7 +659,7 @@ func socketioTests_BitcoinType(t *testing.T, ts *httptest.Server) { { name: "getDetailedTransaction", req: socketioReq{"getDetailedTransaction", []interface{}{"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71"}}, - want: `{"result":{"hex":"","height":225494,"blockTimestamp":22549400001,"version":0,"hash":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","inputs":[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","outputIndex":0,"script":"","sequence":0,"address":"mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX","satoshis":317283951061},{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","outputIndex":1,"script":"","sequence":0,"address":"2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS","satoshis":1}],"inputSatoshis":317283951062,"outputs":[{"satoshis":118641975500,"script":"76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac","address":"mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC"},{"satoshis":198641975500,"script":"76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac","address":"mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP"}],"outputSatoshis":317283951000,"feeSatoshis":62}}`, + want: `{"result":{"hex":"","height":225494,"blockTimestamp":22549400001,"version":0,"hash":"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71","inputs":[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","outputIndex":0,"script":"","sequence":0,"address":"mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX","satoshis":317283951061},{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","outputIndex":1,"script":"","sequence":0,"address":"2MzmAKayJmja784jyHvRUW1bXPget1csRRG","satoshis":1}],"inputSatoshis":317283951062,"outputs":[{"satoshis":118641975500,"script":"a91495e9fbe306449c991d314afe3c3567d5bf78efd287","address":"2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu"},{"satoshis":198641975500,"script":"76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac","address":"mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP"}],"outputSatoshis":317283951000,"feeSatoshis":62}}`, }, { name: "sendTransaction", diff --git a/tests/dbtestdata/dbtestdata.go b/tests/dbtestdata/dbtestdata.go index 36a55c9b..e2f62f05 100644 --- a/tests/dbtestdata/dbtestdata.go +++ b/tests/dbtestdata/dbtestdata.go @@ -16,14 +16,16 @@ const ( TxidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" TxidB2T4 = "fdd824a780cbb718eeb766eb05d83fdefc793a27082cd5e67f856d69798cf7db" + Xpub = "upub5E1xjDmZ7Hhej6LPpS8duATdKXnRYui7bDYj6ehfFGzWDZtmCmQkZhc3Zb7kgRLtHWd16QFxyP86JKL3ShZEBFX88aciJ3xyocuyhZZ8g6q" + Addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac Addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac Addr3 = "mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw" // 76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac - Addr4 = "2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS" // a9144a21db08fb6882cb152e1ff06780a430740f770487 + Addr4 = "2MzmAKayJmja784jyHvRUW1bXPget1csRRG" // a91452724c5178682f70e0ba31c6ec0633755a3b41d987, xpub m/49'/1'/33'/0/0 Addr5 = "2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1" // a914e921fc4912a315078f370d959f2c4f7b6d2a683c87 Addr6 = "mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX" // 76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac Addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac - Addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac + Addr8 = "2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu" // a91495e9fbe306449c991d314afe3c3567d5bf78efd287, xpub m/49'/1'/33'/1/3 Addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac AddrA = "mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj" // 76a914d03c0d863d189b23b061a95ad32940b65837609f88ac ) From 593247c3642e19f9e56c7caebe445321ddb64aa9 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 13 Feb 2019 18:34:31 +0100 Subject: [PATCH 20/26] Fix public interface xpub test --- server/public_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/public_test.go b/server/public_test.go index db6854f1..6f2be7f1 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -332,8 +332,8 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { ``, ``, ``, - `
3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71
mined Tue, 21 Aug 2018 15:45:23 CEST
`, - `
effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75
mined Tue, 21 Aug 2018 15:27:01 CEST
`, + `
3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71
`, + `
effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75
`, ``, ``, }, From 0546a2609d7e870741af06b04501ebaed103799f Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 13 Feb 2019 19:43:16 +0100 Subject: [PATCH 21/26] Count xpub transactions regardless of filter --- api/xpub.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/api/xpub.go b/api/xpub.go index 5a3a2052..0ad492a0 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -325,7 +325,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get var ( txc xpubTxids txmMap map[string]*Tx - txcMap map[string]struct{} + txCount int txs []*Tx txids []string pg Paging @@ -397,18 +397,25 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } } } - txCount := int(data.txCountEstimate) if option >= TxidHistory { - txcMap = make(map[string]struct{}) + txcMap := make(map[string]bool) txc = make(xpubTxids, 0, 32) for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { ad := &da[i] for _, txid := range ad.txids { - _, foundTx := txcMap[txid.txid] - if !foundTx && (useTxids == nil || useTxids(&txid, ad)) { - txcMap[txid.txid] = struct{}{} - txc = append(txc, txid) + added, foundTx := txcMap[txid.txid] + // count txs regardless of filter but only once + if !foundTx { + txCount++ + } + // add tx only once + if !added { + add := useTxids == nil || useTxids(&txid, ad) + txcMap[txid.txid] = add + if add { + txc = append(txc, txid) + } } } } @@ -441,6 +448,8 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get txs = append(txs, tx) } } + } else { + txCount = int(data.txCountEstimate) } totalTokens := 0 var tokens []Token From 88d9e09ad4f84b00479dec71be0e60114cecf038 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 14 Feb 2019 14:53:35 +0100 Subject: [PATCH 22/26] Highlight own ethereum addresses in explorer like in xpub explorer --- server/public.go | 27 ++++++++++++------- static/templates/txdetail.html | 4 +-- static/templates/txdetail_ethereumtype.html | 30 ++++++++++----------- static/templates/xpub.html | 2 +- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/server/public.go b/server/public.go index 7b507c02..40376e33 100644 --- a/server/public.go +++ b/server/public.go @@ -416,6 +416,7 @@ func (s *PublicServer) parseTemplates() []*template.Template { "formatAmountWithDecimals": formatAmountWithDecimals, "setTxToTemplateData": setTxToTemplateData, "isOwnAddress": isOwnAddress, + "isOwnAddresses": isOwnAddresses, } var createTemplate func(filenames ...string) *template.Template if s.debug { @@ -504,19 +505,25 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { return td } -// returns true if addresses are "own", +// returns true if address is "own", // i.e. either the address of the address detail or belonging to the xpub -func isOwnAddress(td *TemplateData, addresses []string) bool { - if len(addresses) == 1 { - a := addresses[0] - if a == td.AddrStr { +func isOwnAddress(td *TemplateData, a string) bool { + if a == td.AddrStr { + return true + } + if td.Address != nil && td.Address.XPubAddresses != nil { + if _, found := td.Address.XPubAddresses[a]; found { return true } - if td.Address != nil && td.Address.XPubAddresses != nil { - if _, found := td.Address.XPubAddresses[a]; found { - return true - } - } + } + return false +} + +// returns true if addresses are "own", +// i.e. either the address of the address detail or belonging to the xpub +func isOwnAddresses(td *TemplateData, addresses []string) bool { + if len(addresses) == 1 { + return isOwnAddress(td, addresses[0]) } return false } diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 57bde9c9..6b3a3f82 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -14,7 +14,7 @@
{{$t.Name}} {{formatAmount $t.BalanceSat}} {{$cs}} {{$t.Transfers}}{{$t.Contract}}{{$t.Path}}
Show all XPUB addressesShow all XPUB addresses
{{$addr.Txs}}
Total XPUB AddressesUsed XPUB Addresses {{$addr.TotalTokens}}
{{if $data.NonZeroBalanceTokens}}XPUB Addresses with Balance{{else}}XPUB Addresses{{end}} From b8672a0fffb8e12c51ddb94d82d9c155dd50f1ad Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 13 Feb 2019 18:10:29 +0100 Subject: [PATCH 19/26] Add public interface test for xpub functionality --- server/public_test.go | 68 ++++++++++++++++++++++++++++++++-- tests/dbtestdata/dbtestdata.go | 6 ++- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/server/public_test.go b/server/public_test.go index a6feca10..db6854f1 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -60,7 +60,13 @@ func setupRocksDB(t *testing.T, parser bchain.BlockChainParser) (*db.RocksDB, *c func setupPublicHTTPServer(t *testing.T) (*PublicServer, string) { parser := btc.NewBitcoinParser( btc.GetChainParams("test"), - &btc.Configuration{BlockAddressesToKeep: 1}) + &btc.Configuration{ + BlockAddressesToKeep: 1, + XPubMagic: 70617039, + XPubMagicSegwitP2sh: 71979618, + XPubMagicSegwitNative: 73342198, + Slip44: 1, + }) d, is, path := setupRocksDB(t, parser) // setup internal state and match BestHeight to test data @@ -314,6 +320,24 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { ``, }, }, + { + name: "explorerSearch xpub", + r: newGetRequest(ts.URL + "/search?q=" + dbtestdata.Xpub), + status: http.StatusOK, + contentType: "text/html; charset=utf-8", + body: []string{ + `Fake Coin Explorer`, + `

XPUB 1186.419755 FAKE

upub5E1xjDmZ7Hhej6LPpS8duATdKXnRYui7bDYj6ehfFGzWDZtmCmQkZhc3Zb7kgRLtHWd16QFxyP86JKL3ShZEBFX88aciJ3xyocuyhZZ8g6q
`, + `		Total Received1186.41975501 FAKETotal Sent0.00000001 FAKEUsed XPUB Addresses22N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu1186.419755 FAKE1m/49'/1'/33'/1/32NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu10.00009876 FAKE ➡Total Sent0.00000001 FAKEUsed XPUB Addresses22N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu1186.419755 FAKE1m/49'/1'/33'/1/32NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu10.00009876 FAKE ➡
{{- range $vin := $tx.Vin -}} - +
{{- if $vin.Txid -}} ➡  @@ -49,7 +49,7 @@ {{- range $vout := $tx.Vout -}} - +
{{- range $a := $vout.Addresses -}} diff --git a/static/templates/txdetail_ethereumtype.html b/static/templates/txdetail_ethereumtype.html index ec760955..bb71a8cc 100644 --- a/static/templates/txdetail_ethereumtype.html +++ b/static/templates/txdetail_ethereumtype.html @@ -1,4 +1,4 @@ -{{define "txdetail"}}{{$cs := .CoinShortcut}}{{$addr := .AddrStr}}{{$tx := .Tx}} +{{define "txdetail"}}{{$cs := .CoinShortcut}}{{$addr := .AddrStr}}{{$tx := .Tx}}{{$data := .}}
@@ -11,18 +11,18 @@
-
+
{{- range $vin := $tx.Vin -}} - + @@ -41,18 +41,18 @@
-
+
{{- range $a := $vin.Addresses -}} - + {{if and (ne $a $addr) $vin.Searchable}}{{$a}}{{else}}{{$a}}{{end}} {{- else -}} - Unparsed address + Unparsed address {{- end -}}
{{- range $vout := $tx.Vout -}} - + @@ -76,12 +76,12 @@ {{- range $erc20 := $tx.TokenTransfers -}}
-
+
{{- range $a := $vout.Addresses -}} - + {{- if and (ne $a $addr) $vout.Searchable}}{{$a}}{{else}}{{$a}}{{- end -}} {{- else -}} - Unparsed address + Unparsed address {{- end -}}
- + @@ -94,12 +94,12 @@
-
+
- {{if ne $erc20.From $addr}}{{$erc20.From}}{{else}}{{$erc20.From}}{{end}} + {{if ne $erc20.From $addr}}{{$erc20.From}}{{else}}{{$erc20.From}}{{end}}
- + diff --git a/static/templates/xpub.html b/static/templates/xpub.html index a4b1f57f..d67c76a3 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -29,7 +29,7 @@ - {{- if $addr.Tokens -}} + {{- if or $addr.Tokens $addr.TotalTokens -}}
- {{if ne $erc20.To $addr}}{{$erc20.To}}{{else}}{{$erc20.To}}{{end}} + {{if ne $erc20.To $addr}}{{$erc20.To}}{{else}}{{$erc20.To}}{{end}}
Used XPUB Addresses {{$addr.TotalTokens}}
{{if $data.NonZeroBalanceTokens}}XPUB Addresses with Balance{{else}}XPUB Addresses{{end}} From c7808b87d50f86035d406a89ea8d7c6b62d63d02 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 14 Feb 2019 14:54:26 +0100 Subject: [PATCH 23/26] Bump blockbook to version 0.2.1 --- configs/environ.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/environ.json b/configs/environ.json index d161716f..6171a41b 100644 --- a/configs/environ.json +++ b/configs/environ.json @@ -1,5 +1,5 @@ { - "version": "0.2.0", + "version": "0.2.1", "backend_install_path": "/opt/coins/nodes", "backend_data_path": "/opt/coins/data", "blockbook_install_path": "/opt/coins/blockbook", From e1eadda6bfb157157c049eade52d3bdd3787696f Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 25 Feb 2019 14:09:24 +0100 Subject: [PATCH 24/26] Make upper limit for gap in xpub address derivation --- api/xpub.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/xpub.go b/api/xpub.go index 0ad492a0..b257c9cb 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -15,6 +15,7 @@ import ( const xpubLen = 111 const defaultAddressesGap = 20 +const maxAddressesGap = 10000 const txInput = 1 const txOutput = 2 @@ -244,6 +245,9 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAdd ) if gap <= 0 { gap = defaultAddressesGap + } else if gap > maxAddressesGap { + // limit the maximum gap to protect against unreasonably big values that could cause high load of the server + gap = maxAddressesGap } // gap is increased one as there must be gap of empty addresses before the derivation is stopped gap++ From 46001a9fa5f36e453b88c0074f7c82b6e684092b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 26 Feb 2019 16:27:28 +0100 Subject: [PATCH 25/26] Evict old cached xpubs --- api/types.go | 1 + api/xpub.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/api/types.go b/api/types.go index 0d1f1b4f..7b4c6034 100644 --- a/api/types.go +++ b/api/types.go @@ -12,6 +12,7 @@ import ( const maxUint32 = ^uint32(0) const maxInt = int(^uint(0) >> 1) +const maxInt64 = int64(^uint64(0) >> 1) // GetAddressOption specifies what data returns GetAddress api call type GetAddressOption int diff --git a/api/xpub.go b/api/xpub.go index b257c9cb..e577f8fa 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -20,6 +20,9 @@ const maxAddressesGap = 10000 const txInput = 1 const txOutput = 2 +const xpubCacheSize = 512 +const xpubCacheExpirationSeconds = 7200 + var cachedXpubs = make(map[string]xpubData) var cachedXpubsMux sync.Mutex @@ -46,6 +49,7 @@ type xpubAddress struct { type xpubData struct { gap int + accessed int64 basePath string dataHeight uint32 dataHash string @@ -234,6 +238,28 @@ func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeInd } } +func evictXpubCacheItems() { + var oldestKey string + oldest := maxInt64 + now := time.Now().Unix() + count := 0 + for k, v := range cachedXpubs { + if v.accessed+xpubCacheExpirationSeconds < now { + delete(cachedXpubs, k) + count++ + } + if v.accessed < oldest { + oldestKey = k + oldest = v.accessed + } + } + if oldestKey != "" && oldest+xpubCacheExpirationSeconds >= now { + delete(cachedXpubs, oldestKey) + count++ + } + glog.Info("Evicted ", count, " items from xpub cache, oldest item accessed at ", time.Unix(oldest, 0), ", cache size ", len(cachedXpubs)) +} + func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*xpubData, uint32, error) { if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { return nil, 0, ErrUnsupportedXpub @@ -309,7 +335,11 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAdd } } } + data.accessed = time.Now().Unix() cachedXpubsMux.Lock() + if len(cachedXpubs) >= xpubCacheSize { + evictXpubCacheItems() + } cachedXpubs[xpub] = data cachedXpubsMux.Unlock() return &data, bestheight, nil From 7b590d9958e5c7f692c16432fc5d4ca577bc3854 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 28 Feb 2019 15:07:07 +0100 Subject: [PATCH 26/26] Unify AccountDetails levels in GetAccount and GetXpub api calls --- api/types.go | 26 +++++----- api/worker.go | 102 +++++++++++++++++++------------------ api/xpub.go | 34 +++++++------ server/public.go | 14 ++--- server/websocket.go | 12 +++-- static/test-websocket.html | 1 + 6 files changed, 99 insertions(+), 90 deletions(-) diff --git a/api/types.go b/api/types.go index 7b4c6034..44751c3d 100644 --- a/api/types.go +++ b/api/types.go @@ -14,20 +14,22 @@ const maxUint32 = ^uint32(0) const maxInt = int(^uint(0) >> 1) const maxInt64 = int64(^uint64(0) >> 1) -// GetAddressOption specifies what data returns GetAddress api call -type GetAddressOption int +// AccountDetails specifies what data returns GetAddress and GetXpub calls +type AccountDetails int const ( - // Basic - only that address is indexed and some basic info - Basic GetAddressOption = iota - // Tokens - basic info + tokens - Tokens - // TxidHistory - basic + tokens + txids, subject to paging - TxidHistory - // TxHistoryLight - basic + tokens + easily obtained tx data (not requiring request to backend), subject to paging - TxHistoryLight - // TxHistory - basic + tokens + full tx data, subject to paging - TxHistory + // AccountDetailsBasic - only that address is indexed and some basic info + AccountDetailsBasic AccountDetails = iota + // AccountDetailsTokens - basic info + tokens + AccountDetailsTokens + // AccountDetailsTokenBalances - basic info + token with balance + AccountDetailsTokenBalances + // AccountDetailsTxidHistory - basic + token balances + txids, subject to paging + AccountDetailsTxidHistory + // AccountDetailsTxHistoryLight - basic + tokens + easily obtained tx data (not requiring requests to backend), subject to paging + AccountDetailsTxHistoryLight + // AccountDetailsTxHistory - basic + tokens + full tx data, subject to paging + AccountDetailsTxHistory ) // ErrUnsupportedXpub is returned when coin type does not support xpub address derivation or provided string is not an xpub diff --git a/api/worker.go b/api/worker.go index 80e19871..d912105e 100644 --- a/api/worker.go +++ b/api/worker.go @@ -479,7 +479,7 @@ func computePaging(count, page, itemsOnPage int) (Paging, int, int, int) { }, from, to, page } -func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescriptor, option GetAddressOption, filter *AddressFilter) (*db.AddrBalance, []Token, *bchain.Erc20Contract, uint64, int, int, error) { +func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescriptor, details AccountDetails, filter *AddressFilter) (*db.AddrBalance, []Token, *bchain.Erc20Contract, uint64, int, int, error) { var ( ba *db.AddrBalance tokens []Token @@ -516,53 +516,55 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto return nil, nil, nil, 0, 0, 0, NewAPIError(fmt.Sprintf("Invalid contract filter, %v", err), true) } } - tokens = make([]Token, len(ca.Contracts)) - var j int - for i, c := range ca.Contracts { - if len(filterDesc) > 0 { - if !bytes.Equal(filterDesc, c.Contract) { - continue + if details > AccountDetailsBasic { + tokens = make([]Token, len(ca.Contracts)) + var j int + for i, c := range ca.Contracts { + if len(filterDesc) > 0 { + if !bytes.Equal(filterDesc, c.Contract) { + continue + } + // filter only transactions of this contract + filter.Vout = i + 1 } - // filter only transactions of this contract - filter.Vout = i + 1 - } - validContract := true - ci, err := w.chain.EthereumTypeGetErc20ContractInfo(c.Contract) - if err != nil { - return nil, nil, nil, 0, 0, 0, errors.Annotatef(err, "EthereumTypeGetErc20ContractInfo %v", c.Contract) - } - if ci == nil { - ci = &bchain.Erc20Contract{} - addresses, _, _ := w.chainParser.GetAddressesFromAddrDesc(c.Contract) - if len(addresses) > 0 { - ci.Contract = addresses[0] - ci.Name = addresses[0] - } - validContract = false - } - // do not read contract balances etc in case of Basic option - if option != Basic && validContract { - b, err = w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, c.Contract) + validContract := true + ci, err := w.chain.EthereumTypeGetErc20ContractInfo(c.Contract) if err != nil { - // return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract) - glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err) + return nil, nil, nil, 0, 0, 0, errors.Annotatef(err, "EthereumTypeGetErc20ContractInfo %v", c.Contract) } - } else { - b = nil + if ci == nil { + ci = &bchain.Erc20Contract{} + addresses, _, _ := w.chainParser.GetAddressesFromAddrDesc(c.Contract) + if len(addresses) > 0 { + ci.Contract = addresses[0] + ci.Name = addresses[0] + } + validContract = false + } + // do not read contract balances etc in case of Basic option + if details >= AccountDetailsTokenBalances && validContract { + b, err = w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, c.Contract) + if err != nil { + // return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract) + glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err) + } + } else { + b = nil + } + tokens[j] = Token{ + Type: ERC20TokenType, + BalanceSat: (*Amount)(b), + Contract: ci.Contract, + Name: ci.Name, + Symbol: ci.Symbol, + Transfers: int(c.Txs), + Decimals: ci.Decimals, + ContractIndex: strconv.Itoa(i + 1), + } + j++ } - tokens[j] = Token{ - Type: ERC20TokenType, - BalanceSat: (*Amount)(b), - Contract: ci.Contract, - Name: ci.Name, - Symbol: ci.Symbol, - Transfers: int(c.Txs), - Decimals: ci.Decimals, - ContractIndex: strconv.Itoa(i + 1), - } - j++ + tokens = tokens[:j] } - tokens = tokens[:j] ci, err = w.chain.EthereumTypeGetErc20ContractInfo(addrDesc) if err != nil { return nil, nil, nil, 0, 0, 0, err @@ -582,11 +584,11 @@ func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescripto return ba, tokens, ci, n, nonContractTxs, totalResults, nil } -func (w *Worker) txFromTxid(txid string, bestheight uint32, option GetAddressOption) (*Tx, error) { +func (w *Worker) txFromTxid(txid string, bestheight uint32, option AccountDetails) (*Tx, error) { var tx *Tx var err error // only ChainBitcoinType supports TxHistoryLight - if option == TxHistoryLight && w.chainType == bchain.ChainBitcoinType { + if option == AccountDetailsTxHistoryLight && w.chainType == bchain.ChainBitcoinType { ta, err := w.db.GetTxAddresses(txid) if err != nil { return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) @@ -632,7 +634,7 @@ func (w *Worker) getAddrDescAndNormalizeAddress(address string) (bchain.AddressD } // GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { +func (w *Worker) GetAddress(address string, page int, txsOnPage int, option AccountDetails, filter *AddressFilter) (*Address, error) { start := time.Now() page-- if page < 0 { @@ -700,9 +702,9 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) if page == 0 { - if option == TxidHistory { + if option == AccountDetailsTxidHistory { txids = append(txids, tx.Txid) - } else if option >= TxHistoryLight { + } else if option >= AccountDetailsTxHistoryLight { txs = append(txs, tx) } } @@ -711,7 +713,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA } } // get tx history if requested by option or check mempool if there are some transactions for a new address - if option >= TxidHistory { + if option >= AccountDetailsTxidHistory { txc, err := w.getAddressTxids(addrDesc, false, filter, (page+1)*txsOnPage) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc) @@ -731,7 +733,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetA } for i := from; i < to; i++ { txid := txc[i] - if option == TxidHistory { + if option == AccountDetailsTxidHistory { txids = append(txids, txid) } else { tx, err := w.txFromTxid(txid, bestheight, option) diff --git a/api/xpub.go b/api/xpub.go index e577f8fa..3419b945 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -212,7 +212,7 @@ func (w *Worker) xpubScanAddresses(xpub string, data *xpubData, addresses []xpub return lastUsed, addresses, nil } -func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeIndex int, index int) Token { +func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeIndex int, index int, option AccountDetails) Token { a, _, _ := w.chainParser.GetAddressesFromAddrDesc(ad.addrDesc) var address string if len(a) > 0 { @@ -221,10 +221,12 @@ func (w *Worker) tokenFromXpubAddress(data *xpubData, ad *xpubAddress, changeInd var balance, totalReceived, totalSent *big.Int var transfers int if ad.balance != nil { - balance = &ad.balance.BalanceSat - totalSent = &ad.balance.SentSat - totalReceived = ad.balance.ReceivedSat() transfers = int(ad.balance.Txs) + if option >= AccountDetailsTokenBalances { + balance = &ad.balance.BalanceSat + totalSent = &ad.balance.SentSat + totalReceived = ad.balance.ReceivedSat() + } } return Token{ Type: XPUBAddressTokenType, @@ -260,7 +262,7 @@ func evictXpubCacheItems() { glog.Info("Evicted ", count, " items from xpub cache, oldest item accessed at ", time.Unix(oldest, 0), ", cache size ", len(cachedXpubs)) } -func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*xpubData, uint32, error) { +func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option AccountDetails, filter *AddressFilter, gap int) (*xpubData, uint32, error) { if w.chainType != bchain.ChainBitcoinType || len(xpub) != xpubLen { return nil, 0, ErrUnsupportedXpub } @@ -325,7 +327,7 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAdd return nil, 0, err } } - if option >= TxidHistory { + if option >= AccountDetailsTxidHistory { for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { if err = w.xpubCheckAndLoadTxids(&da[i], filter, bestheight, (page+1)*txsOnPage); err != nil { @@ -346,7 +348,7 @@ func (w *Worker) getXpubData(xpub string, page int, txsOnPage int, option GetAdd } // GetXpubAddress computes address value and gets transactions for given address -func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter, gap int) (*Address, error) { +func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option AccountDetails, filter *AddressFilter, gap int) (*Address, error) { start := time.Now() page-- if page < 0 { @@ -419,9 +421,9 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get uBalSat.Add(&uBalSat, tx.getAddrVoutValue(ad.addrDesc)) uBalSat.Sub(&uBalSat, tx.getAddrVinValue(ad.addrDesc)) if page == 0 && !foundTx && (useTxids == nil || useTxids(&txid, ad)) { - if option == TxidHistory { + if option == AccountDetailsTxidHistory { txids = append(txids, tx.Txid) - } else if option >= TxHistoryLight { + } else if option >= AccountDetailsTxHistoryLight { txs = append(txs, tx) } } @@ -431,7 +433,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get } } } - if option >= TxidHistory { + if option >= AccountDetailsTxidHistory { txcMap := make(map[string]bool) txc = make(xpubTxids, 0, 32) for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { @@ -472,7 +474,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get // get confirmed transactions for i := from; i < to; i++ { xpubTxid := &txc[i] - if option == TxidHistory { + if option == AccountDetailsTxidHistory { txids = append(txids, xpubTxid.txid) } else { tx, err := w.txFromTxid(xpubTxid.txid, bestheight, option) @@ -488,7 +490,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get totalTokens := 0 var tokens []Token var xpubAddresses map[string]struct{} - if option != Basic { + if option > AccountDetailsBasic { tokens = make([]Token, 0, 4) xpubAddresses = make(map[string]struct{}) } @@ -498,8 +500,8 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get if ad.balance != nil { totalTokens++ } - if option != Basic { - token := w.tokenFromXpubAddress(data, ad, ci, i) + if option > AccountDetailsBasic { + token := w.tokenFromXpubAddress(data, ad, ci, i, option) if filter.TokenLevel == TokenDetailDiscovered || filter.TokenLevel == TokenDetailUsed && ad.balance != nil || filter.TokenLevel == TokenDetailNonzeroBalance && ad.balance != nil && !IsZeroBigInt(&ad.balance.BalanceSat) { @@ -533,7 +535,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Get // GetXpubUtxo returns unspent outputs for given xpub func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, error) { start := time.Now() - data, _, err := w.getXpubData(xpub, 0, 1, Basic, &AddressFilter{ + data, _, err := w.getXpubData(xpub, 0, 1, AccountDetailsBasic, &AddressFilter{ Vout: AddressFilterVoutOff, OnlyConfirmed: onlyConfirmed, }, gap) @@ -556,7 +558,7 @@ func (w *Worker) GetXpubUtxo(xpub string, onlyConfirmed bool, gap int) (Utxos, e return nil, err } if len(utxos) > 0 { - t := w.tokenFromXpubAddress(data, ad, ci, i) + t := w.tokenFromXpubAddress(data, ad, ci, i, AccountDetailsTokens) for j := range utxos { a := &utxos[j] a.Address = t.Name diff --git a/server/public.go b/server/public.go index 40376e33..3b21a4ff 100644 --- a/server/public.go +++ b/server/public.go @@ -590,7 +590,7 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( } } } - address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsOnPage, api.AccountDetailsTxHistoryLight, &api.AddressFilter{Vout: fn}) if err != nil { return errorTpl, nil, err } @@ -607,7 +607,7 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( return addressTpl, data, nil } -func (s *PublicServer) getXpubAddress(r *http.Request, xpub string, pageSize int, option api.GetAddressOption) (*api.Address, api.TokenDetailLevel, error) { +func (s *PublicServer) getXpubAddress(r *http.Request, xpub string, pageSize int, option api.AccountDetails) (*api.Address, api.TokenDetailLevel, error) { var fn = api.AddressFilterVoutOff page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { @@ -650,7 +650,7 @@ func (s *PublicServer) explorerXpub(w http.ResponseWriter, r *http.Request) (tpl var err error s.metrics.ExplorerViews.With(common.Labels{"action": "xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - address, tokenLevel, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsOnPage, api.TxHistoryLight) + address, tokenLevel, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsOnPage, api.AccountDetailsTxHistoryLight) if err != nil { return errorTpl, nil, err } @@ -730,7 +730,7 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t var err error s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() if len(q) > 0 { - address, err = s.api.GetXpubAddress(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, 0) + address, err = s.api.GetXpubAddress(q, 0, 1, api.AccountDetailsBasic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}, 0) if err == nil { http.Redirect(w, r, joinURL("/xpub/", address.AddrStr), 302) return noTpl, nil, nil @@ -745,7 +745,7 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t http.Redirect(w, r, joinURL("/tx/", tx.Txid), 302) return noTpl, nil, nil } - address, err = s.api.GetAddress(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + address, err = s.api.GetAddress(q, 0, 1, api.AccountDetailsBasic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) if err == nil { http.Redirect(w, r, joinURL("/address/", address.AddrStr), 302) return noTpl, nil, nil @@ -907,7 +907,7 @@ func (s *PublicServer) apiAddress(r *http.Request, apiVersion int) (interface{}, if ec != nil { page = 0 } - address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsInAPI, api.AccountDetailsTxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } @@ -920,7 +920,7 @@ func (s *PublicServer) apiXpub(r *http.Request, apiVersion int) (interface{}, er var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-xpub"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - address, _, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsInAPI, api.TxidHistory) + address, _, err = s.getXpubAddress(r, r.URL.Path[i+1:], txsInAPI, api.AccountDetailsTxidHistory) if err == nil && apiVersion == apiV1 { return s.api.AddressToV1(address), nil } diff --git a/server/websocket.go b/server/websocket.go index aa1d746e..e71b0c7e 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -347,16 +347,18 @@ func unmarshalGetAccountInfoRequest(params []byte) (*accountInfoReq, error) { } func (s *WebsocketServer) getAccountInfo(req *accountInfoReq) (res *api.Address, err error) { - var opt api.GetAddressOption + var opt api.AccountDetails switch req.Details { case "tokens": - opt = api.Tokens + opt = api.AccountDetailsTokens + case "tokenBalances": + opt = api.AccountDetailsTokenBalances case "txids": - opt = api.TxidHistory + opt = api.AccountDetailsTxidHistory case "txs": - opt = api.TxHistory + opt = api.AccountDetailsTxHistory default: - opt = api.Basic + opt = api.AccountDetailsBasic } filter := api.AddressFilter{ FromHeight: uint32(req.FromHeight), diff --git a/static/test-websocket.html b/static/test-websocket.html index ed28c05f..37274254 100644 --- a/static/test-websocket.html +++ b/static/test-websocket.html @@ -305,6 +305,7 @@