RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Golomb filter of scriptPubKeys for mempool txs

Browse Source
This commit is contained in:
Martin Boehm 2023-05-08 23:11:49 +02:00
parent d83d501272
commit c8c7e08a18
7 changed files with 147 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
bchain/coins/btc/bitcoinrpc.go
View File

@ -23,16 +23,17 @@ import (
// BitcoinRPC is an interface to JSON-RPC bitcoind service.
type BitcoinRPC struct {
*bchain.BaseChain
client http.Client
rpcURL string
user string
password string
Mempool *bchain.MempoolBitcoinType
ParseBlocks bool
pushHandler func(bchain.NotificationType)
mq *bchain.MQ
ChainConfig *Configuration
RPCMarshaler RPCMarshaler
client http.Client
rpcURL string
user string
password string
Mempool *bchain.MempoolBitcoinType
ParseBlocks bool
pushHandler func(bchain.NotificationType)
mq *bchain.MQ
ChainConfig *Configuration
RPCMarshaler RPCMarshaler
golombFilterP uint8
}
// Configuration represents json config file
@ -60,6 +61,7 @@ type Configuration struct {
AlternativeEstimateFee string `json:"alternative_estimate_fee,omitempty"`
AlternativeEstimateFeeParams string `json:"alternative_estimate_fee_params,omitempty"`
MinimumCoinbaseConfirmations int `json:"minimumCoinbaseConfirmations,omitempty"`
GolombFilterP uint8 `json:"golomb_filter_p,omitempty"`
}
// NewBitcoinRPC returns new BitcoinRPC instance.
@ -96,15 +98,16 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT
}
s := &BitcoinRPC{
BaseChain: &bchain.BaseChain{},
client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport},
rpcURL: c.RPCURL,
user: c.RPCUser,
password: c.RPCPass,
ParseBlocks: c.Parse,
ChainConfig: &c,
pushHandler: pushHandler,
RPCMarshaler: JSONMarshalerV2{},
BaseChain: &bchain.BaseChain{},
client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport},
rpcURL: c.RPCURL,
user: c.RPCUser,
password: c.RPCPass,
ParseBlocks: c.Parse,
ChainConfig: &c,
pushHandler: pushHandler,
RPCMarshaler: JSONMarshalerV2{},
golombFilterP: c.GolombFilterP,
}
return s, nil
@ -150,7 +153,7 @@ func (b *BitcoinRPC) Initialize() error {
// CreateMempool creates mempool if not already created, however does not initialize it
func (b *BitcoinRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
if b.Mempool == nil {
b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers)
b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers, b.golombFilterP)
}
return b.Mempool, nil
}

40
bchain/mempool_bitcoin_type.go
View File

@ -1,10 +1,12 @@
package bchain
import (
"encoding/hex"
"math/big"
"time"
"github.com/golang/glog"
"github.com/martinboehm/btcutil/gcs"
)
type chanInputPayload struct {
@ -18,11 +20,14 @@ type MempoolBitcoinType struct {
chanTxid chan string
chanAddrIndex chan txidio
AddrDescForOutpoint AddrDescForOutpointFunc
golombFilterP uint8
golombFilterM uint64
}
// NewMempoolBitcoinType creates new mempool handler.
// For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process
func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *MempoolBitcoinType {
func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8) *MempoolBitcoinType {
golombFilterM := uint64(1 << golombFilterP)
m := &MempoolBitcoinType{
BaseMempool: BaseMempool{
chain: chain,
@ -31,6 +36,8 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *Mempo
},
chanTxid: make(chan string, 1),
chanAddrIndex: make(chan txidio, 1),
golombFilterP: golombFilterP,
golombFilterM: golombFilterM,
}
for i := 0; i < workers; i++ {
go func(i int) {
@ -91,6 +98,34 @@ func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrInd
}
func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx) string {
filterData := make([][]byte, 0)
for i := range mtx.Vin {
vin := &mtx.Vin[i]
filterData = append(filterData, vin.AddrDesc)
}
for i := range mtx.Vout {
vout := &mtx.Vout[i]
b, err := hex.DecodeString(vout.ScriptPubKey.Hex)
if err == nil {
filterData = append(filterData, b)
}
}
if len(filterData) == 0 {
return ""
}
b, _ := hex.DecodeString(mtx.Txid)
if len(b) < gcs.KeySize {
return ""
}
filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), filterData)
if err != nil {
glog.Error("Cannot create golomb filter for ", mtx.Txid, ", ", err)
}
fb, _ := filter.Bytes()
return hex.EncodeToString(fb)
}
func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan chanInputPayload, chanResult chan *addrIndex) ([]addrIndex, bool) {
tx, err := m.chain.GetTransactionForMempool(txid)
if err != nil {
@ -142,6 +177,9 @@ func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan chanInputPay
io = append(io, *ai)
}
}
if m.golombFilterP > 0 {
mtx.GolombFilter = m.computeGolombFilter(mtx)
}
if m.OnNewTx != nil {
m.OnNewTx(mtx)
}

79
bchain/mempool_bitcoin_type_test.go Normal file
View File

@ -0,0 +1,79 @@
package bchain
import (
"encoding/hex"
"testing"
"github.com/martinboehm/btcutil/gcs"
)
func hexToBytes(h string) []byte {
b, _ := hex.DecodeString(h)
return b
}
func TestMempoolBitcoinType_computeGolombFilter(t *testing.T) {
randomScript := hexToBytes("a914ff074800343a81ada8fe86c2d5d5a0e55b93dd7a87")
m := &MempoolBitcoinType{
golombFilterP: 20,
golombFilterM: uint64(1 << 20),
}
tests := []struct {
name string
mtx MempoolTx
want string
}{
{
name: "taproot",
mtx: MempoolTx{
Txid: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
Vin: []MempoolVin{
{
// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
AddrDesc: hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
},
},
Vout: []Vout{
{
ScriptPubKey: ScriptPubKey{
Hex: "5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f",
Addresses: []string{
"bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav",
},
},
},
},
},
want: "35dddcce5d60",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := m.computeGolombFilter(&tt.mtx); got != tt.want {
t.Errorf("MempoolBitcoinType.computeGolombFilter() = %v, want %v", got, tt.want)
}
// check that the vin script matches the filter
b, _ := hex.DecodeString(tt.mtx.Txid)
filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), [][]byte{tt.mtx.Vin[0].AddrDesc})
if err != nil {
t.Errorf("gcs.BuildGCSFilter() unexpected error %v", err)
}
match, err := filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), tt.mtx.Vin[0].AddrDesc)
if err != nil {
t.Errorf("filter.Match vin[0] unexpected error %v", err)
}
if match != true {
t.Errorf("filter.Match vin[0] expected true, got false")
}
// check that a random script does not match the filter
match, err = filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), randomScript)
if err != nil {
t.Errorf("filter.Match randomScript unexpected error %v", err)
}
if match != false {
t.Errorf("filter.Match randomScript expected false, got true")
}
})
}
}

1
bchain/types.go
View File

@ -113,6 +113,7 @@ type MempoolTx struct {
Blocktime int64 `json:"blocktime,omitempty"`
TokenTransfers TokenTransfers `json:"-"`
CoinSpecificData interface{} `json:"-"`
GolombFilter string `json:"-"`
}
// TokenType - type of token

2
go.mod
View File

@ -38,6 +38,7 @@ require (
github.com/Groestlcoin/go-groestl-hash v0.0.0-20181012171753-790653ac190c // indirect
github.com/PiRK/cashaddr-converter v0.0.0-20220121162910-c6cb45163b29 // indirect
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
github.com/aead/siphash v1.0.1 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
@ -65,6 +66,7 @@ require (
github.com/holiman/uint256 v1.2.0 // indirect
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b // indirect
github.com/kkdai/bstream v0.0.0-20171226095907-f71540b9dfdc // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect

2
go.sum
View File

@ -5,6 +5,7 @@ github.com/PiRK/cashaddr-converter v0.0.0-20220121162910-c6cb45163b29 h1:B11Brye
github.com/PiRK/cashaddr-converter v0.0.0-20220121162910-c6cb45163b29/go.mod h1:+39XiGr9m9TPY49sG4XIH5CVaRxHGFWT0U4MOY6dy3o=
github.com/VictoriaMetrics/fastcache v1.10.0 h1:5hDJnLsKLpnUEToub7ETuRu8RCkb40woBZAUiKonXzY=
github.com/VictoriaMetrics/fastcache v1.10.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8=
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI=
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
@ -138,6 +139,7 @@ github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVE
github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b h1:Rrp0ByJXEjhREMPGTt3aWYjoIsUGCbt21ekbeJcTWv0=
github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/kkdai/bstream v0.0.0-20171226095907-f71540b9dfdc h1:I1QApI4r4SG8Hh45H0yRjVnThWRn1oOwod76rrAe5KE=
github.com/kkdai/bstream v0.0.0-20171226095907-f71540b9dfdc/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=

2
tests/dbtestdata/fakechain.go
View File

@ -19,7 +19,7 @@ func NewFakeBlockChain(parser bchain.BlockChainParser) (bchain.BlockChain, error
}
func (c *fakeBlockChain) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
return bchain.NewMempoolBitcoinType(chain, 1, 1), nil
return bchain.NewMempoolBitcoinType(chain, 1, 1, 0), nil
}
func (c *fakeBlockChain) Initialize() error {