Golomb filter of scriptPubKeys for mempool txs

bchain/coins/btc/bitcoinrpc.go

@@ -23,16 +23,17 @@ import (
 // BitcoinRPC is an interface to JSON-RPC bitcoind service.
 type BitcoinRPC struct {
 	*bchain.BaseChain
-	client       http.Client
+	client        http.Client
-	rpcURL       string
+	rpcURL        string
-	user         string
+	user          string
-	password     string
+	password      string
-	Mempool      *bchain.MempoolBitcoinType
+	Mempool       *bchain.MempoolBitcoinType
-	ParseBlocks  bool
+	ParseBlocks   bool
-	pushHandler  func(bchain.NotificationType)
+	pushHandler   func(bchain.NotificationType)
-	mq           *bchain.MQ
+	mq            *bchain.MQ
-	ChainConfig  *Configuration
+	ChainConfig   *Configuration
-	RPCMarshaler RPCMarshaler
+	RPCMarshaler  RPCMarshaler
+	golombFilterP uint8
 }
 
 // Configuration represents json config file
@@ -60,6 +61,7 @@ type Configuration struct {
 	AlternativeEstimateFee       string `json:"alternative_estimate_fee,omitempty"`
 	AlternativeEstimateFeeParams string `json:"alternative_estimate_fee_params,omitempty"`
 	MinimumCoinbaseConfirmations int    `json:"minimumCoinbaseConfirmations,omitempty"`
+	GolombFilterP                uint8  `json:"golomb_filter_p,omitempty"`
 }
 
 // NewBitcoinRPC returns new BitcoinRPC instance.
@@ -96,15 +98,16 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT
 	}
 
 	s := &BitcoinRPC{
-		BaseChain:    &bchain.BaseChain{},
+		BaseChain:     &bchain.BaseChain{},
-		client:       http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport},
+		client:        http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport},
-		rpcURL:       c.RPCURL,
+		rpcURL:        c.RPCURL,
-		user:         c.RPCUser,
+		user:          c.RPCUser,
-		password:     c.RPCPass,
+		password:      c.RPCPass,
-		ParseBlocks:  c.Parse,
+		ParseBlocks:   c.Parse,
-		ChainConfig:  &c,
+		ChainConfig:   &c,
-		pushHandler:  pushHandler,
+		pushHandler:   pushHandler,
-		RPCMarshaler: JSONMarshalerV2{},
+		RPCMarshaler:  JSONMarshalerV2{},
+		golombFilterP: c.GolombFilterP,
 	}
 
 	return s, nil
@@ -150,7 +153,7 @@ func (b *BitcoinRPC) Initialize() error {
 // CreateMempool creates mempool if not already created, however does not initialize it
 func (b *BitcoinRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
 	if b.Mempool == nil {
-		b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers)
+		b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers, b.golombFilterP)
 	}
 	return b.Mempool, nil
 }

bchain/mempool_bitcoin_type.go

@@ -1,10 +1,12 @@
 package bchain
 
 import (
+	"encoding/hex"
 	"math/big"
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/martinboehm/btcutil/gcs"
 )
 
 type chanInputPayload struct {
@@ -18,11 +20,14 @@ type MempoolBitcoinType struct {
 	chanTxid            chan string
 	chanAddrIndex       chan txidio
 	AddrDescForOutpoint AddrDescForOutpointFunc
+	golombFilterP       uint8
+	golombFilterM       uint64
 }
 
 // NewMempoolBitcoinType creates new mempool handler.
 // For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process
-func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *MempoolBitcoinType {
+func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8) *MempoolBitcoinType {
+	golombFilterM := uint64(1 << golombFilterP)
 	m := &MempoolBitcoinType{
 		BaseMempool: BaseMempool{
 			chain:        chain,
@@ -31,6 +36,8 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *Mempo
 		},
 		chanTxid:      make(chan string, 1),
 		chanAddrIndex: make(chan txidio, 1),
+		golombFilterP: golombFilterP,
+		golombFilterM: golombFilterM,
 	}
 	for i := 0; i < workers; i++ {
 		go func(i int) {
@@ -91,6 +98,34 @@ func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrInd
 
 }
 
+func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx) string {
+	filterData := make([][]byte, 0)
+	for i := range mtx.Vin {
+		vin := &mtx.Vin[i]
+		filterData = append(filterData, vin.AddrDesc)
+	}
+	for i := range mtx.Vout {
+		vout := &mtx.Vout[i]
+		b, err := hex.DecodeString(vout.ScriptPubKey.Hex)
+		if err == nil {
+			filterData = append(filterData, b)
+		}
+	}
+	if len(filterData) == 0 {
+		return ""
+	}
+	b, _ := hex.DecodeString(mtx.Txid)
+	if len(b) < gcs.KeySize {
+		return ""
+	}
+	filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), filterData)
+	if err != nil {
+		glog.Error("Cannot create golomb filter for ", mtx.Txid, ", ", err)
+	}
+	fb, _ := filter.Bytes()
+	return hex.EncodeToString(fb)
+}
+
 func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan chanInputPayload, chanResult chan *addrIndex) ([]addrIndex, bool) {
 	tx, err := m.chain.GetTransactionForMempool(txid)
 	if err != nil {
@@ -142,6 +177,9 @@ func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan chanInputPay
 			io = append(io, *ai)
 		}
 	}
+	if m.golombFilterP > 0 {
+		mtx.GolombFilter = m.computeGolombFilter(mtx)
+	}
 	if m.OnNewTx != nil {
 		m.OnNewTx(mtx)
 	}

bchain/mempool_bitcoin_type_test.go

@@ -0,0 +1,79 @@
+package bchain
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/martinboehm/btcutil/gcs"
+)
+
+func hexToBytes(h string) []byte {
+	b, _ := hex.DecodeString(h)
+	return b
+}
+
+func TestMempoolBitcoinType_computeGolombFilter(t *testing.T) {
+	randomScript := hexToBytes("a914ff074800343a81ada8fe86c2d5d5a0e55b93dd7a87")
+	m := &MempoolBitcoinType{
+		golombFilterP: 20,
+		golombFilterM: uint64(1 << 20),
+	}
+	tests := []struct {
+		name string
+		mtx  MempoolTx
+		want string
+	}{
+		{
+			name: "taproot",
+			mtx: MempoolTx{
+				Txid: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
+				Vin: []MempoolVin{
+					{
+						// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
+						AddrDesc: hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
+					},
+				},
+				Vout: []Vout{
+					{
+						ScriptPubKey: ScriptPubKey{
+							Hex: "5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f",
+							Addresses: []string{
+								"bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav",
+							},
+						},
+					},
+				},
+			},
+			want: "35dddcce5d60",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := m.computeGolombFilter(&tt.mtx); got != tt.want {
+				t.Errorf("MempoolBitcoinType.computeGolombFilter() = %v, want %v", got, tt.want)
+			}
+			// check that the vin script matches the filter
+			b, _ := hex.DecodeString(tt.mtx.Txid)
+			filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), [][]byte{tt.mtx.Vin[0].AddrDesc})
+			if err != nil {
+				t.Errorf("gcs.BuildGCSFilter() unexpected error %v", err)
+			}
+			match, err := filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), tt.mtx.Vin[0].AddrDesc)
+			if err != nil {
+				t.Errorf("filter.Match vin[0] unexpected error %v", err)
+			}
+			if match != true {
+				t.Errorf("filter.Match vin[0] expected true, got false")
+			}
+			// check that a random script does not match the filter
+			match, err = filter.Match(*(*[gcs.KeySize]byte)(b[:gcs.KeySize]), randomScript)
+			if err != nil {
+				t.Errorf("filter.Match randomScript unexpected error %v", err)
+			}
+			if match != false {
+				t.Errorf("filter.Match randomScript expected false, got true")
+			}
+
+		})
+	}
+}

bchain/types.go

@@ -113,6 +113,7 @@ type MempoolTx struct {
 	Blocktime        int64          `json:"blocktime,omitempty"`
 	TokenTransfers   TokenTransfers `json:"-"`
 	CoinSpecificData interface{}    `json:"-"`
+	GolombFilter     string         `json:"-"`
 }
 
 // TokenType - type of token

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/Groestlcoin/go-groestl-hash v0.0.0-20181012171753-790653ac190c // indirect
 	github.com/PiRK/cashaddr-converter v0.0.0-20220121162910-c6cb45163b29 // indirect
 	github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
+	github.com/aead/siphash v1.0.1 // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
@@ -65,6 +66,7 @@ require (
 	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b // indirect
+	github.com/kkdai/bstream v0.0.0-20171226095907-f71540b9dfdc // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -5,6 +5,7 @@ github.com/PiRK/cashaddr-converter v0.0.0-20220121162910-c6cb45163b29 h1:B11Brye
 github.com/PiRK/cashaddr-converter v0.0.0-20220121162910-c6cb45163b29/go.mod h1:+39XiGr9m9TPY49sG4XIH5CVaRxHGFWT0U4MOY6dy3o=
 github.com/VictoriaMetrics/fastcache v1.10.0 h1:5hDJnLsKLpnUEToub7ETuRu8RCkb40woBZAUiKonXzY=
 github.com/VictoriaMetrics/fastcache v1.10.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8=
+github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI=
 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0=
@@ -138,6 +139,7 @@ github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVE
 github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b h1:Rrp0ByJXEjhREMPGTt3aWYjoIsUGCbt21ekbeJcTWv0=
 github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/kkdai/bstream v0.0.0-20171226095907-f71540b9dfdc h1:I1QApI4r4SG8Hh45H0yRjVnThWRn1oOwod76rrAe5KE=
 github.com/kkdai/bstream v0.0.0-20171226095907-f71540b9dfdc/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=

tests/dbtestdata/fakechain.go

@@ -19,7 +19,7 @@ func NewFakeBlockChain(parser bchain.BlockChainParser) (bchain.BlockChain, error
 }
 
 func (c *fakeBlockChain) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
-	return bchain.NewMempoolBitcoinType(chain, 1, 1), nil
+	return bchain.NewMempoolBitcoinType(chain, 1, 1, 0), nil
 }
 
 func (c *fakeBlockChain) Initialize() error {