RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use only taproot scripts for mempool golomb filter

Browse Source
This commit is contained in:
Martin Boehm 2023-05-09 09:06:13 +02:00
parent c8c7e08a18
commit 3ab5e636ff
6 changed files with 284 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
bchain/coins/btc/bitcoinrpc.go
View File

@ -23,17 +23,18 @@ import (
// BitcoinRPC is an interface to JSON-RPC bitcoind service.
type BitcoinRPC struct {
*bchain.BaseChain
client http.Client
rpcURL string
user string
password string
Mempool *bchain.MempoolBitcoinType
ParseBlocks bool
pushHandler func(bchain.NotificationType)
mq *bchain.MQ
ChainConfig *Configuration
RPCMarshaler RPCMarshaler
golombFilterP uint8
client http.Client
rpcURL string
user string
password string
Mempool *bchain.MempoolBitcoinType
ParseBlocks bool
pushHandler func(bchain.NotificationType)
mq *bchain.MQ
ChainConfig *Configuration
RPCMarshaler RPCMarshaler
golombFilterP uint8
golombFilterScripts string
}
// Configuration represents json config file
@ -62,6 +63,7 @@ type Configuration struct {
AlternativeEstimateFeeParams string `json:"alternative_estimate_fee_params,omitempty"`
MinimumCoinbaseConfirmations int `json:"minimumCoinbaseConfirmations,omitempty"`
GolombFilterP uint8 `json:"golomb_filter_p,omitempty"`
GolombFilterScripts string `json:"golomb_filter_scripts,omitempty"`
}
// NewBitcoinRPC returns new BitcoinRPC instance.
@ -98,16 +100,17 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT
}
s := &BitcoinRPC{
BaseChain: &bchain.BaseChain{},
client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport},
rpcURL: c.RPCURL,
user: c.RPCUser,
password: c.RPCPass,
ParseBlocks: c.Parse,
ChainConfig: &c,
pushHandler: pushHandler,
RPCMarshaler: JSONMarshalerV2{},
golombFilterP: c.GolombFilterP,
BaseChain: &bchain.BaseChain{},
client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport},
rpcURL: c.RPCURL,
user: c.RPCUser,
password: c.RPCPass,
ParseBlocks: c.Parse,
ChainConfig: &c,
pushHandler: pushHandler,
RPCMarshaler: JSONMarshalerV2{},
golombFilterP: c.GolombFilterP,
golombFilterScripts: c.GolombFilterScripts,
}
return s, nil
@ -153,7 +156,7 @@ func (b *BitcoinRPC) Initialize() error {
// CreateMempool creates mempool if not already created, however does not initialize it
func (b *BitcoinRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
if b.Mempool == nil {
b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers, b.golombFilterP)
b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers, b.golombFilterP, b.golombFilterScripts)
}
return b.Mempool, nil
}

42
bchain/mempool_bitcoin_type.go
View File

@ -14,6 +14,11 @@ type chanInputPayload struct {
index int
}
type golombFilterScriptsType int
const golombFilterScriptsAll = golombFilterScriptsType(0)
const golombFilterScriptsTaproot = golombFilterScriptsType(1)
// MempoolBitcoinType is mempool handle.
type MempoolBitcoinType struct {
BaseMempool
@ -22,11 +27,22 @@ type MempoolBitcoinType struct {
AddrDescForOutpoint AddrDescForOutpointFunc
golombFilterP uint8
golombFilterM uint64
golombFilterScripts golombFilterScriptsType
}
// NewMempoolBitcoinType creates new mempool handler.
// For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process
func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8) *MempoolBitcoinType {
func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8, golombFilterScripts string) *MempoolBitcoinType {
var filterScripts golombFilterScriptsType
switch golombFilterScripts {
case "":
filterScripts = golombFilterScriptsAll
case "taproot":
filterScripts = golombFilterScriptsTaproot
default:
glog.Error("Invalid golombFilterScripts ", golombFilterScripts, ", switching off golomb filter")
golombFilterP = 0
}
golombFilterM := uint64(1 << golombFilterP)
m := &MempoolBitcoinType{
BaseMempool: BaseMempool{
@ -34,10 +50,11 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golomb
txEntries: make(map[string]txEntry),
addrDescToTx: make(map[string][]Outpoint),
},
chanTxid: make(chan string, 1),
chanAddrIndex: make(chan txidio, 1),
golombFilterP: golombFilterP,
golombFilterM: golombFilterM,
chanTxid: make(chan string, 1),
chanAddrIndex: make(chan txidio, 1),
golombFilterP: golombFilterP,
golombFilterM: golombFilterM,
golombFilterScripts: filterScripts,
}
for i := 0; i < workers; i++ {
go func(i int) {
@ -98,17 +115,28 @@ func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrInd
}
func isTaproot(addrDesc AddressDescriptor) bool {
if len(addrDesc) == 34 && addrDesc[0] == 0x51 && addrDesc[1] == 0x20 {
return true
}
return false
}
func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx) string {
filterData := make([][]byte, 0)
for i := range mtx.Vin {
vin := &mtx.Vin[i]
filterData = append(filterData, vin.AddrDesc)
if m.golombFilterScripts == golombFilterScriptsAll || (m.golombFilterScripts == golombFilterScriptsTaproot && isTaproot(vin.AddrDesc)) {
filterData = append(filterData, vin.AddrDesc)
}
}
for i := range mtx.Vout {
vout := &mtx.Vout[i]
b, err := hex.DecodeString(vout.ScriptPubKey.Hex)
if err == nil {
filterData = append(filterData, b)
if m.golombFilterScripts == golombFilterScriptsAll || (m.golombFilterScripts == golombFilterScriptsTaproot && isTaproot(b)) {
filterData = append(filterData, b)
}
}
}
if len(filterData) == 0 {

173
bchain/mempool_bitcoin_type_test.go
View File

@ -12,19 +12,22 @@ func hexToBytes(h string) []byte {
return b
}
func TestMempoolBitcoinType_computeGolombFilter(t *testing.T) {
func TestMempoolBitcoinType_computeGolombFilter_taproot(t *testing.T) {
randomScript := hexToBytes("a914ff074800343a81ada8fe86c2d5d5a0e55b93dd7a87")
m := &MempoolBitcoinType{
golombFilterP: 20,
golombFilterM: uint64(1 << 20),
golombFilterP: 20,
golombFilterM: uint64(1 << 20),
golombFilterScripts: golombFilterScriptsTaproot,
}
tests := []struct {
name string
N uint32
mtx MempoolTx
want string
}{
{
name: "taproot",
N: 2,
mtx: MempoolTx{
Txid: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
Vin: []MempoolVin{
@ -46,34 +49,154 @@ func TestMempoolBitcoinType_computeGolombFilter(t *testing.T) {
},
want: "35dddcce5d60",
},
{
name: "taproot multiple",
N: 7,
mtx: MempoolTx{
Txid: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
Vin: []MempoolVin{
{
// bc1pp3752xgfy39w30kggy8vvn0u68x8afwqmq6p96jzr8ffrcvjxgrqrny93y
AddrDesc: hexToBytes("51200c7d451909244ae8bec8410ec64dfcd1cc7ea5c0d83412ea4219d291e1923206"),
},
{
// bc1p5ldsz3zxnjxrwf4xluf4qu7u839c204ptacwe2k0vzfk8s63mwts3njuwr
AddrDesc: hexToBytes("5120a7db0144469c8c3726a6ff135073dc3c4b853ea15f70ecaacf609363c351db97"),
},
{
// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
AddrDesc: hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
},
},
Vout: []Vout{
{
ScriptPubKey: ScriptPubKey{
Hex: "51209ab20580f77e7cd676f896fc1794f7e8061efc1ce7494f2bb16205262aa12bdb",
Addresses: []string{
"bc1pn2eqtq8h0e7dvahcjm7p098haqrpalquuay572a3vgzjv24p90dszxzg40",
},
},
},
{
ScriptPubKey: ScriptPubKey{
Hex: "5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f",
Addresses: []string{
"bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav",
},
},
},
{
ScriptPubKey: ScriptPubKey{
Hex: "51201341e5a58314d89bcf5add2b2a68f109add5efb1ae774fa33c612da311f25904",
Addresses: []string{
"bc1pzdq7tfvrznvfhn66m54j5683pxkatma34em5lgeuvyk6xy0jtyzqjt48z3",
},
},
},
{
ScriptPubKey: ScriptPubKey{
Hex: "512042b2d5c032b68220bfd6d4e26bc015129e168e87e22af743ffdc736708b7d342",
Addresses: []string{
"bc1pg2edtspjk6pzp07k6n3xhsq4z20pdr58ug40wsllm3ekwz9h6dpq77lhu9",
},
},
},
},
},
want: "1143e4ad12730965a5247ac15db8c81c89b0bc",
},
{
name: "partial taproot",
N: 1,
mtx: MempoolTx{
Txid: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
Vin: []MempoolVin{
{
// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
AddrDesc: hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
},
},
Vout: []Vout{
{
ScriptPubKey: ScriptPubKey{
Hex: "00145f997834e1135e893b7707ba1b12bcb8d74b821d",
Addresses: []string{
"bc1qt7vhsd8pzd0gjwmhq7apky4uhrt5hqsa2y58nl",
},
},
},
},
},
want: "1aeee8",
},
{
name: "no taproot",
N: 0,
mtx: MempoolTx{
Txid: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
Vin: []MempoolVin{
{
// 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ
AddrDesc: hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"),
},
},
Vout: []Vout{
{
ScriptPubKey: ScriptPubKey{
Hex: "00145f997834e1135e893b7707ba1b12bcb8d74b821d",
Addresses: []string{
"bc1qt7vhsd8pzd0gjwmhq7apky4uhrt5hqsa2y58nl",
},
},
},
},
},
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := m.computeGolombFilter(&tt.mtx); got != tt.want {
got := m.computeGolombFilter(&tt.mtx)
if got != tt.want {
t.Errorf("MempoolBitcoinType.computeGolombFilter() = %v, want %v", got, tt.want)
}
// check that the vin script matches the filter
b, _ := hex.DecodeString(tt.mtx.Txid)
filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), [][]byte{tt.mtx.Vin[0].AddrDesc})
if err != nil {
t.Errorf("gcs.BuildGCSFilter() unexpected error %v", err)
if got != "" {
// build the filter from computed value
filter, err := gcs.FromBytes(tt.N, m.golombFilterP, m.golombFilterM, hexToBytes(got))
if err != nil {
t.Errorf("gcs.BuildGCSFilter() unexpected error %v", err)
}
// check that the vin scripts match the filter
b, _ := hex.DecodeString(tt.mtx.Txid)
for i := range tt.mtx.Vin {
match, err := filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), tt.mtx.Vin[i].AddrDesc)
if err != nil {
t.Errorf("filter.Match vin[%d] unexpected error %v", i, err)
}
if match != isTaproot(tt.mtx.Vin[i].AddrDesc) {
t.Errorf("filter.Match vin[%d] got %v, want %v", i, match, isTaproot(tt.mtx.Vin[i].AddrDesc))
}
}
// check that the vout scripts match the filter
for i := range tt.mtx.Vout {
s := hexToBytes(tt.mtx.Vout[i].ScriptPubKey.Hex)
match, err := filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), s)
if err != nil {
t.Errorf("filter.Match vout[%d] unexpected error %v", i, err)
}
if match != isTaproot(s) {
t.Errorf("filter.Match vout[%d] got %v, want %v", i, match, isTaproot(s))
}
}
// check that a random script does not match the filter
match, err := filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), randomScript)
if err != nil {
t.Errorf("filter.Match randomScript unexpected error %v", err)
}
if match != false {
t.Errorf("filter.Match randomScript got true, want false")
}
}
match, err := filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), tt.mtx.Vin[0].AddrDesc)
if err != nil {
t.Errorf("filter.Match vin[0] unexpected error %v", err)
}
if match != true {
t.Errorf("filter.Match vin[0] expected true, got false")
}
// check that a random script does not match the filter
match, err = filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), randomScript)
if err != nil {
t.Errorf("filter.Match randomScript unexpected error %v", err)
}
if match != false {
t.Errorf("filter.Match randomScript expected false, got true")
}
})
}
}

4
configs/coins/bitcoin.json
View File

@ -67,7 +67,9 @@
"alternative_estimate_fee_params": "{\"url\": \"https://whatthefee.io/data.json\", \"periodSeconds\": 60}",
"fiat_rates": "coingecko",
"fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH",
"fiat_rates_params": "{\"coin\": \"bitcoin\", \"periodSeconds\": 900}"
"fiat_rates_params": "{\"coin\": \"bitcoin\", \"periodSeconds\": 900}",
"golomb_filter_p": 20,
"golomb_filter_scripts": "taproot"
}
}
},

141
configs/coins/bitcoin_testnet.json
View File

@ -1,73 +1,76 @@
{
"coin": {
"name": "Testnet",
"shortcut": "TEST",
"label": "Bitcoin Testnet",
"alias": "bitcoin_testnet"
},
"ports": {
"backend_rpc": 18030,
"backend_message_queue": 48330,
"blockbook_internal": 19030,
"blockbook_public": 19130
},
"ipc": {
"rpc_url_template": "http://127.0.0.1:{{.Ports.BackendRPC}}",
"rpc_user": "rpc",
"rpc_pass": "rpc",
"rpc_timeout": 25,
"message_queue_binding_template": "tcp://127.0.0.1:{{.Ports.BackendMessageQueue}}"
},
"backend": {
"package_name": "backend-bitcoin-testnet",
"package_revision": "satoshilabs-1",
"system_user": "bitcoin",
"version": "24.0.1",
"binary_url": "https://bitcoincore.org/bin/bitcoin-core-24.0.1/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz",
"verification_type": "sha256",
"verification_source": "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf",
"extract_command": "tar -C backend --strip 1 -xf",
"exclude_files": ["bin/bitcoin-qt"],
"exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bitcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid",
"logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet3/*.log",
"postinst_script_template": "",
"service_type": "forking",
"service_additional_params_template": "",
"protect_memory": true,
"mainnet": false,
"server_config_file": "bitcoin.conf",
"client_config_file": "bitcoin_client.conf",
"additional_params": {
"deprecatedrpc": "estimatefee"
"coin": {
"name": "Testnet",
"shortcut": "TEST",
"label": "Bitcoin Testnet",
"alias": "bitcoin_testnet"
},
"platforms": {
"arm64": {
"binary_url": "https://bitcoincore.org/bin/bitcoin-core-23.0/bitcoin-23.0-aarch64-linux-gnu.tar.gz",
"verification_source": "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb"
}
"ports": {
"backend_rpc": 18030,
"backend_message_queue": 48330,
"blockbook_internal": 19030,
"blockbook_public": 19130
},
"ipc": {
"rpc_url_template": "http://127.0.0.1:{{.Ports.BackendRPC}}",
"rpc_user": "rpc",
"rpc_pass": "rpc",
"rpc_timeout": 25,
"message_queue_binding_template": "tcp://127.0.0.1:{{.Ports.BackendMessageQueue}}"
},
"backend": {
"package_name": "backend-bitcoin-testnet",
"package_revision": "satoshilabs-1",
"system_user": "bitcoin",
"version": "24.0.1",
"binary_url": "https://bitcoincore.org/bin/bitcoin-core-24.0.1/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz",
"verification_type": "sha256",
"verification_source": "49df6e444515d457ea0b885d66f521f2a26ca92ccf73d5296082e633544253bf",
"extract_command": "tar -C backend --strip 1 -xf",
"exclude_files": ["bin/bitcoin-qt"],
"exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/bitcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid",
"logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/testnet3/*.log",
"postinst_script_template": "",
"service_type": "forking",
"service_additional_params_template": "",
"protect_memory": true,
"mainnet": false,
"server_config_file": "bitcoin.conf",
"client_config_file": "bitcoin_client.conf",
"additional_params": {
"deprecatedrpc": "estimatefee"
},
"platforms": {
"arm64": {
"binary_url": "https://bitcoincore.org/bin/bitcoin-core-23.0/bitcoin-23.0-aarch64-linux-gnu.tar.gz",
"verification_source": "06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb"
}
}
},
"blockbook": {
"package_name": "blockbook-bitcoin-testnet",
"system_user": "blockbook-bitcoin",
"internal_binding_template": ":{{.Ports.BlockbookInternal}}",
"public_binding_template": ":{{.Ports.BlockbookPublic}}",
"explorer_url": "",
"additional_params": "-enablesubnewtx -extendedindex",
"block_chain": {
"parse": true,
"mempool_workers": 8,
"mempool_sub_workers": 2,
"block_addresses_to_keep": 300,
"xpub_magic": 70617039,
"xpub_magic_segwit_p2sh": 71979618,
"xpub_magic_segwit_native": 73342198,
"slip44": 1,
"additional_params": {
"golomb_filter_p": 20,
"golomb_filter_scripts": "taproot"
}
}
},
"meta": {
"package_maintainer": "IT",
"package_maintainer_email": "it@satoshilabs.com"
}
},
"blockbook": {
"package_name": "blockbook-bitcoin-testnet",
"system_user": "blockbook-bitcoin",
"internal_binding_template": ":{{.Ports.BlockbookInternal}}",
"public_binding_template": ":{{.Ports.BlockbookPublic}}",
"explorer_url": "",
"additional_params": "-enablesubnewtx -extendedindex",
"block_chain": {
"parse": true,
"mempool_workers": 8,
"mempool_sub_workers": 2,
"block_addresses_to_keep": 300,
"xpub_magic": 70617039,
"xpub_magic_segwit_p2sh": 71979618,
"xpub_magic_segwit_native": 73342198,
"slip44": 1,
"additional_params": {}
}
},
"meta": {
"package_maintainer": "IT",
"package_maintainer_email": "it@satoshilabs.com"
}
}

2
tests/dbtestdata/fakechain.go
View File

@ -19,7 +19,7 @@ func NewFakeBlockChain(parser bchain.BlockChainParser) (bchain.BlockChain, error
}
func (c *fakeBlockChain) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
return bchain.NewMempoolBitcoinType(chain, 1, 1, 0), nil
return bchain.NewMempoolBitcoinType(chain, 1, 1, 0, ""), nil
}
func (c *fakeBlockChain) Initialize() error {