Use golomb config in block sync, refactor

bchain/golomb.go

@@ -0,0 +1,102 @@
+package bchain
+
+import (
+	"encoding/hex"
+
+	"github.com/golang/glog"
+	"github.com/juju/errors"
+	"github.com/martinboehm/btcutil/gcs"
+)
+
+type FilterScriptsType int
+
+const (
+	FilterScriptsInvalid = FilterScriptsType(iota)
+	FilterScriptsAll
+	FilterScriptsTaproot
+)
+
+// GolombFilter is computing golomb filter of address descriptors
+type GolombFilter struct {
+	Enabled           bool
+	p                 uint8
+	key               string
+	filterScripts     string
+	filterScriptsType FilterScriptsType
+	filterData        [][]byte
+	uniqueData        map[string]struct{}
+}
+
+// NewGolombFilter initializes the GolombFilter handler
+func NewGolombFilter(p uint8, filterScripts string, key string) (*GolombFilter, error) {
+	if p == 0 {
+		return &GolombFilter{Enabled: false}, nil
+	}
+	gf := GolombFilter{
+		Enabled:           true,
+		p:                 p,
+		key:               key,
+		filterScripts:     filterScripts,
+		filterScriptsType: filterScriptsToScriptsType(filterScripts),
+		filterData:        make([][]byte, 0),
+		uniqueData:        make(map[string]struct{}),
+	}
+	// only taproot and all is supported
+	if gf.filterScriptsType == FilterScriptsInvalid {
+		return nil, errors.Errorf("Invalid/unsupported filterScripts parameter %s", filterScripts)
+	}
+	return &gf, nil
+}
+
+// AddAddrDesc adds taproot address descriptor to the data for the filter
+func (f *GolombFilter) AddAddrDesc(ad AddressDescriptor) {
+	if f.filterScriptsType == FilterScriptsTaproot && !ad.IsTaproot() {
+		return
+	}
+	if len(ad) == 0 {
+		return
+	}
+	s := string(ad)
+	if _, found := f.uniqueData[s]; !found {
+		f.filterData = append(f.filterData, ad)
+		f.uniqueData[s] = struct{}{}
+	}
+}
+
+// Compute computes golomb filter from the data
+func (f *GolombFilter) Compute() []byte {
+	m := uint64(1 << uint64(f.p))
+
+	if len(f.filterData) == 0 {
+		return nil
+	}
+
+	b, _ := hex.DecodeString(f.key)
+	if len(b) < gcs.KeySize {
+		return nil
+	}
+
+	filter, err := gcs.BuildGCSFilter(f.p, m, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), f.filterData)
+	if err != nil {
+		glog.Error("Cannot create golomb filter for ", f.key, ", ", err)
+		return nil
+	}
+
+	fb, err := filter.NBytes()
+	if err != nil {
+		glog.Error("Error getting NBytes from golomb filter for ", f.key, ", ", err)
+		return nil
+	}
+
+	return fb
+}
+
+func filterScriptsToScriptsType(filterScripts string) FilterScriptsType {
+	switch filterScripts {
+	case "":
+		return FilterScriptsAll
+	case "taproot":
+		return FilterScriptsTaproot
+	}
+	return FilterScriptsInvalid
+}

bchain/golomb_test.go

@@ -0,0 +1,117 @@
+// //go:build unittest
+
+package bchain
+
+import (
+	"encoding/hex"
+	"testing"
+)
+
+func TestGolombFilter(t *testing.T) {
+	tests := []struct {
+		name               string
+		p                  uint8
+		filterScripts      string
+		key                string
+		addressDescriptors [][]byte
+		wantError          bool
+		wantEnabled        bool
+		want               string
+	}{
+		{
+			name:          "taproot",
+			p:             20,
+			filterScripts: "taproot",
+			key:           "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
+			addressDescriptors: [][]byte{
+				// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
+				hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
+				// bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav
+				hexToBytes("5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f"),
+				// 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ
+				hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"),
+			},
+			wantEnabled: true,
+			wantError:   false,
+			want:        "0235dddcce5d60",
+		},
+		{
+			name:          "taproot p=21",
+			p:             21,
+			filterScripts: "taproot",
+			key:           "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
+			addressDescriptors: [][]byte{
+				// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
+				hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
+				// bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav
+				hexToBytes("5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f"),
+				// 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ
+				hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"),
+			},
+			wantEnabled: true,
+			wantError:   false,
+			want:        "0235ddda672eb0",
+		},
+		{
+			name:          "all",
+			p:             20,
+			filterScripts: "",
+			key:           "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2",
+			addressDescriptors: [][]byte{
+				// bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5
+				hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"),
+				// bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav
+				hexToBytes("5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f"),
+				// 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ
+				hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"),
+			},
+			wantEnabled: true,
+			wantError:   false,
+			want:        "0350ccc61ac611976c80",
+		},
+		{
+			name:          "not supported filter",
+			p:             20,
+			filterScripts: "notsupported",
+			wantEnabled:   false,
+			wantError:     true,
+			want:          "",
+		},
+		{
+			name:          "not enabled",
+			p:             0,
+			filterScripts: "",
+			wantEnabled:   false,
+			wantError:     false,
+			want:          "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gf, err := NewGolombFilter(tt.p, tt.filterScripts, tt.key)
+			if err != nil && !tt.wantError {
+				t.Errorf("TestGolombFilter.NewGolombFilter() got unexpected error '%v'", err)
+				return
+			}
+			if err == nil && tt.wantError {
+				t.Errorf("TestGolombFilter.NewGolombFilter() wanted error, got none")
+				return
+			}
+			if gf == nil && tt.wantError {
+				return
+			}
+			if gf.Enabled != tt.wantEnabled {
+				t.Errorf("TestGolombFilter.NewGolombFilter() got gf.Enabled %v, want %v", gf.Enabled, tt.wantEnabled)
+				return
+			}
+			for _, ad := range tt.addressDescriptors {
+				gf.AddAddrDesc(ad)
+			}
+			f := gf.Compute()
+			got := hex.EncodeToString(f)
+			if got != tt.want {
+				t.Errorf("TestGolombFilter Compute() got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

bchain/mempool_bitcoin_type.go

@@ -2,13 +2,11 @@ package bchain
 
 import (
 	"encoding/hex"
-	"errors"
-	"fmt"
 	"math/big"
 	"time"
 
 	"github.com/golang/glog"
-	"github.com/martinboehm/btcutil/gcs"
+	"github.com/juju/errors"
 )
 
 type chanInputPayload struct {
@@ -16,14 +14,6 @@ type chanInputPayload struct {
 	index int
 }
 
-type filterScriptsType int
-
-const (
-	filterScriptsInvalid = filterScriptsType(iota)
-	filterScriptsAll
-	filterScriptsTaproot
-)
-
 // MempoolBitcoinType is mempool handle.
 type MempoolBitcoinType struct {
 	BaseMempool
@@ -31,19 +21,12 @@ type MempoolBitcoinType struct {
 	chanAddrIndex       chan txidio
 	AddrDescForOutpoint AddrDescForOutpointFunc
 	golombFilterP       uint8
-	golombFilterM       uint64
+	filterScripts       string
-	filterScripts       filterScriptsType
 }
 
 // NewMempoolBitcoinType creates new mempool handler.
 // For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process
 func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8, filterScripts string) *MempoolBitcoinType {
-	filterScriptsType := filterScriptsToScriptsType(filterScripts)
-	if filterScriptsType == filterScriptsInvalid {
-		glog.Error("Invalid filterScripts ", filterScripts, ", switching off golomb filter")
-		golombFilterP = 0
-	}
-	golombFilterM := uint64(1 << golombFilterP)
 	m := &MempoolBitcoinType{
 		BaseMempool: BaseMempool{
 			chain:        chain,
@@ -53,8 +36,7 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golomb
 		chanTxid:      make(chan string, 1),
 		chanAddrIndex: make(chan txidio, 1),
 		golombFilterP: golombFilterP,
-		golombFilterM: golombFilterM,
+		filterScripts: filterScripts,
-		filterScripts: filterScriptsType,
 	}
 	for i := 0; i < workers; i++ {
 		go func(i int) {
@@ -81,16 +63,6 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golomb
 	return m
 }
 
-func filterScriptsToScriptsType(filterScripts string) filterScriptsType {
-	switch filterScripts {
-	case "":
-		return filterScriptsAll
-	case "taproot":
-		return filterScriptsTaproot
-	}
-	return filterScriptsInvalid
-}
-
 func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrIndex {
 	var addrDesc AddressDescriptor
 	var value *big.Int
@@ -126,49 +98,20 @@ func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrInd
 }
 
 func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx) string {
-	uniqueScripts := make(map[string]struct{})
+	gf, _ := NewGolombFilter(m.golombFilterP, m.filterScripts, mtx.Txid)
-	filterData := make([][]byte, 0)
+	if gf == nil || !gf.Enabled {
-
+		return ""
-	handleAddrDesc := func(ad AddressDescriptor) {
-		if m.filterScripts == filterScriptsAll || (m.filterScripts == filterScriptsTaproot && ad.IsTaproot()) {
-			if len(ad) == 0 {
-				return
-			}
-			s := string(ad)
-			if _, found := uniqueScripts[s]; !found {
-				filterData = append(filterData, ad)
-				uniqueScripts[s] = struct{}{}
-			}
-		}
 	}
-
 	for _, vin := range mtx.Vin {
-		handleAddrDesc(vin.AddrDesc)
+		gf.AddAddrDesc(vin.AddrDesc)
 	}
 	for _, vout := range mtx.Vout {
 		b, err := hex.DecodeString(vout.ScriptPubKey.Hex)
 		if err == nil {
-			handleAddrDesc(b)
+			gf.AddAddrDesc(b)
 		}
 	}
-
+	fb := gf.Compute()
-	if len(filterData) == 0 {
-		return ""
-	}
-	b, _ := hex.DecodeString(mtx.Txid)
-	if len(b) < gcs.KeySize {
-		return ""
-	}
-	filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), filterData)
-	if err != nil {
-		glog.Error("Cannot create golomb filter for ", mtx.Txid, ", ", err)
-		return ""
-	}
-	fb, err := filter.NBytes()
-	if err != nil {
-		glog.Error("Error getting NBytes from golomb filter for ", mtx.Txid, ", ", err)
-		return ""
-	}
 	return hex.EncodeToString(fb)
 }
 
@@ -295,8 +238,8 @@ func (m *MempoolBitcoinType) Resync() (int, error) {
 
 // GetTxidFilterEntries returns all mempool entries with golomb filter from
 func (m *MempoolBitcoinType) GetTxidFilterEntries(filterScripts string, fromTimestamp uint32) (MempoolTxidFilterEntries, error) {
-	if m.filterScripts != filterScriptsToScriptsType(filterScripts) {
+	if m.filterScripts != filterScripts {
-		return MempoolTxidFilterEntries{}, errors.New(fmt.Sprint("Unsupported script filter ", filterScripts))
+		return MempoolTxidFilterEntries{}, errors.Errorf("Unsupported script filter %s", filterScripts)
 	}
 	m.mux.Lock()
 	entries := make(map[string]string)

bchain/mempool_bitcoin_type_test.go

@@ -16,9 +16,9 @@ func TestMempoolBitcoinType_computeGolombFilter_taproot(t *testing.T) {
 	randomScript := hexToBytes("a914ff074800343a81ada8fe86c2d5d5a0e55b93dd7a87")
 	m := &MempoolBitcoinType{
 		golombFilterP: 20,
-		golombFilterM: uint64(1 << 20),
+		filterScripts: "taproot",
-		filterScripts: filterScriptsTaproot,
 	}
+	golombFilterM := uint64(1 << uint64(m.golombFilterP))
 	tests := []struct {
 		name string
 		mtx  MempoolTx
@@ -200,7 +200,7 @@ func TestMempoolBitcoinType_computeGolombFilter_taproot(t *testing.T) {
 			}
 			if got != "" {
 				// build the filter from computed value
-				filter, err := gcs.FromNBytes(m.golombFilterP, m.golombFilterM, hexToBytes(got))
+				filter, err := gcs.FromNBytes(m.golombFilterP, golombFilterM, hexToBytes(got))
 				if err != nil {
 					t.Errorf("gcs.BuildGCSFilter() unexpected error %v", err)
 				}

db/blockfilter.go

@@ -1,68 +0,0 @@
-package db
-
-import (
-	"encoding/hex"
-
-	"github.com/golang/glog"
-	"github.com/martinboehm/btcutil/gcs"
-	"github.com/trezor/blockbook/bchain"
-)
-
-func computeBlockFilter(allAddrDesc [][]byte, blockHash string, taprootOnly bool) string {
-	// TODO: take these from config - how to access it? From BitcoinRPC?
-	// TODO: these things should probably be an argument to this function,
-	// so it is better testable
-	golombFilterP := uint8(20)
-	golombFilterM := uint64(1 << golombFilterP)
-
-	// TODO: code below is almost a copy-paste from computeGolombFilter,
-	// it might be possible to refactor it into a common function, e.g.
-	// computeGolomb(allAddrDescriptors, P, M, taprootOnly, hashIdentifier) -> filterData
-	// but where to put it?
-
-	uniqueScripts := make(map[string]struct{})
-	filterData := make([][]byte, 0)
-
-	handleAddrDesc := func(ad bchain.AddressDescriptor) {
-		if taprootOnly && !ad.IsTaproot() {
-			return
-		}
-		if len(ad) == 0 {
-			return
-		}
-		s := string(ad)
-		if _, found := uniqueScripts[s]; !found {
-			filterData = append(filterData, ad)
-			uniqueScripts[s] = struct{}{}
-		}
-	}
-
-	for _, ad := range allAddrDesc {
-		handleAddrDesc(ad)
-	}
-
-	if len(filterData) == 0 {
-		return ""
-	}
-
-	b, _ := hex.DecodeString(blockHash)
-	if len(b) < gcs.KeySize {
-		return ""
-	}
-
-	filter, err := gcs.BuildGCSFilter(golombFilterP, golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), filterData)
-	if err != nil {
-		glog.Error("Cannot create golomb filter for ", blockHash, ", ", err)
-		return ""
-	}
-
-	fb, err := filter.NBytes()
-	if err != nil {
-		glog.Error("Error getting NBytes from golomb filter for ", blockHash, ", ", err)
-		return ""
-	}
-
-	// TODO: maybe not returning string but []byte, when we are saving it
-	// as []byte anyway?
-	return hex.EncodeToString(fb)
-}

db/blockfilter_test.go

@@ -1,98 +0,0 @@
-//go:build unittest
-
-package db
-
-import (
-	"math/big"
-	"testing"
-
-	"github.com/trezor/blockbook/tests/dbtestdata"
-)
-
-func TestComputeBlockFilter(t *testing.T) {
-	// TODO: add more (vectorized) tests, with taproot txs
-	// - both taprootOnly=true and taprootOnly=false
-	// - check that decoding with different P does not work
-	allAddrDesc := getallAddrDesc()
-	blockHash := "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"
-	taprootOnly := false
-	got := computeBlockFilter(allAddrDesc, blockHash, taprootOnly)
-	want := "0847a3118f0a689307a375c45c1b02379119579910ee80"
-	if got != want {
-		t.Errorf("computeBlockFilter() failed, expected: %s, got: %s", want, got)
-	}
-}
-
-func getallAddrDesc() [][]byte {
-	allAddrDesc := make([][]byte, 0)
-	parser := bitcoinTestnetParser()
-
-	// TODO: this data is copied exactly, make it common and reuse it
-	ta := &TxAddresses{
-		Height: 12345,
-		VSize:  321,
-		Inputs: []TxInput{
-			{
-				AddrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser),
-				ValueSat: *big.NewInt(9011000000),
-				Txid:     "c50c7ce2f5670fd52de738288299bd854a85ef1bb304f62f35ced1bd49a8a810",
-				Vout:     0,
-			},
-			{
-				AddrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser),
-				ValueSat: *big.NewInt(8011000000),
-				Txid:     "e96672c7fcc8da131427fcea7e841028614813496a56c11e8a6185c16861c495",
-				Vout:     1,
-			},
-			{
-				AddrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser),
-				ValueSat: *big.NewInt(7011000000),
-				Txid:     "ed308c72f9804dfeefdbb483ef8fd1e638180ad81d6b33f4b58d36d19162fa6d",
-				Vout:     134,
-			},
-		},
-		Outputs: []TxOutput{
-			{
-				AddrDesc:    addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser),
-				ValueSat:    *big.NewInt(5011000000),
-				Spent:       true,
-				SpentTxid:   dbtestdata.TxidB1T1,
-				SpentIndex:  0,
-				SpentHeight: 432112345,
-			},
-			{
-				AddrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser),
-				ValueSat: *big.NewInt(6011000000),
-			},
-			{
-				AddrDesc:    addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser),
-				ValueSat:    *big.NewInt(7011000000),
-				Spent:       true,
-				SpentTxid:   dbtestdata.TxidB1T2,
-				SpentIndex:  14231,
-				SpentHeight: 555555,
-			},
-			{
-				AddrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser),
-				ValueSat: *big.NewInt(999900000),
-			},
-			{
-				AddrDesc:    addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser),
-				ValueSat:    *big.NewInt(5000000000),
-				Spent:       true,
-				SpentTxid:   dbtestdata.TxidB2T1,
-				SpentIndex:  674541,
-				SpentHeight: 6666666,
-			},
-		},
-	}
-
-	for _, input := range ta.Inputs {
-		allAddrDesc = append(allAddrDesc, input.AddrDesc)
-	}
-	for _, output := range ta.Outputs {
-		allAddrDesc = append(allAddrDesc, output.AddrDesc)
-	}
-
-	return allAddrDesc
-}

db/bulkconnect.go

@@ -27,7 +27,7 @@ type BulkConnect struct {
 	bulkAddressesCount int
 	ethBlockTxs        []ethBlockTx
 	txAddressesMap     map[string]*TxAddresses
-	blockFilters       map[string]string
+	blockFilters       map[string][]byte
 	balances           map[string]*AddrBalance
 	addressContracts   map[string]*AddrContracts
 	height             uint32
@@ -52,6 +52,7 @@ func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) {
 		txAddressesMap:   make(map[string]*TxAddresses),
 		balances:         make(map[string]*AddrBalance),
 		addressContracts: make(map[string]*AddrContracts),
+		blockFilters:     make(map[string][]byte),
 	}
 	if err := d.SetInconsistentState(true); err != nil {
 		return nil, err
@@ -178,14 +179,20 @@ func (b *BulkConnect) storeBulkBlockFilters(wb *grocksdb.WriteBatch) error {
 			return err
 		}
 	}
-	b.blockFilters = make(map[string]string)
+	b.blockFilters = make(map[string][]byte)
 	return nil
 }
 
 func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs bool) error {
 	addresses := make(addressesMap)
-	allBlockAddrDesc := make([][]byte, 0)
+	gf, err := bchain.NewGolombFilter(b.d.is.BlockGolombFilterP, b.d.is.BlockFilterScripts, block.BlockHeader.Hash)
-	if err := b.d.processAddressesBitcoinType(block, addresses, b.txAddressesMap, b.balances, &allBlockAddrDesc); err != nil {
+	if err != nil {
+		glog.Error("connectBlockBitcoinType golomb filter error ", err)
+		gf = nil
+	} else if gf != nil && !gf.Enabled {
+		gf = nil
+	}
+	if err := b.d.processAddressesBitcoinType(block, addresses, b.txAddressesMap, b.balances, gf); err != nil {
 		return err
 	}
 	var storeAddressesChan, storeBalancesChan chan error
@@ -212,11 +219,9 @@ func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs
 		addresses: addresses,
 	})
 	b.bulkAddressesCount += len(addresses)
-	if b.blockFilters == nil {
+	if gf != nil {
-		b.blockFilters = make(map[string]string) // TODO: where to put this?
+		b.blockFilters[block.BlockHeader.Hash] = gf.Compute()
 	}
-	taprootOnly := true // TODO: take from config
-	b.blockFilters[block.BlockHeader.Hash] = computeBlockFilter(allBlockAddrDesc, block.BlockHeader.Hash, taprootOnly)
 	// open WriteBatch only if going to write
 	if sa || b.bulkAddressesCount > maxBulkAddresses || storeBlockTxs || len(b.blockFilters) > maxBlockFilters {
 		start := time.Now()

db/rocksdb.go

@@ -348,9 +348,15 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error {
 	addresses := make(addressesMap)
 	if chainType == bchain.ChainBitcoinType {
 		txAddressesMap := make(map[string]*TxAddresses)
-		allBlockAddrDesc := make([][]byte, 0)
 		balances := make(map[string]*AddrBalance)
-		if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances, &allBlockAddrDesc); err != nil {
+		gf, err := bchain.NewGolombFilter(d.is.BlockGolombFilterP, d.is.BlockFilterScripts, block.BlockHeader.Hash)
+		if err != nil {
+			glog.Error("ConnectBlock golomb filter error ", err)
+			gf = nil
+		} else if gf != nil && !gf.Enabled {
+			gf = nil
+		}
+		if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances, gf); err != nil {
 			return err
 		}
 		if err := d.storeTxAddresses(wb, txAddressesMap); err != nil {
@@ -362,10 +368,11 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error {
 		if err := d.storeAndCleanupBlockTxs(wb, block); err != nil {
 			return err
 		}
-		taprootOnly := true // TODO: take from config
+		if gf != nil {
-		blockFilter := computeBlockFilter(allBlockAddrDesc, block.BlockHeader.Hash, taprootOnly)
+			blockFilter := gf.Compute()
-		if err := d.storeBlockFilter(wb, block.BlockHeader.Hash, blockFilter); err != nil {
+			if err := d.storeBlockFilter(wb, block.BlockHeader.Hash, blockFilter); err != nil {
-			return err
+				return err
+			}
 		}
 	} else if chainType == bchain.ChainEthereumType {
 		addressContracts := make(map[string]*AddrContracts)
@@ -597,8 +604,7 @@ func (d *RocksDB) GetAndResetConnectBlockStats() string {
 	return s
 }
 
-// TODO: maybe return allBlockAddrDesc from this function instead of taking it as argument
+func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses addressesMap, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance, gf *bchain.GolombFilter) error {
-func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses addressesMap, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance, allBlockAddrDesc *[][]byte) error {
 	blockTxIDs := make([][]byte, len(block.Txs))
 	blockTxAddresses := make([]*TxAddresses, len(block.Txs))
 	// first process all outputs so that inputs can refer to txs in this block
@@ -636,7 +642,9 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add
 				}
 				continue
 			}
-			*allBlockAddrDesc = append(*allBlockAddrDesc, addrDesc) // new addrDesc
+			if gf != nil {
+				gf.AddAddrDesc(addrDesc)
+			}
 			tao.AddrDesc = addrDesc
 			if d.chainParser.IsAddrDescIndexable(addrDesc) {
 				strAddrDesc := string(addrDesc)
@@ -711,7 +719,9 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add
 			if spentOutput.Spent {
 				glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout)
 			}
-			*allBlockAddrDesc = append(*allBlockAddrDesc, spentOutput.AddrDesc) // new addrDesc
+			if gf != nil {
+				gf.AddAddrDesc(spentOutput.AddrDesc)
+			}
 			tai.AddrDesc = spentOutput.AddrDesc
 			tai.ValueSat = spentOutput.ValueSat
 			// mark the output as spent in tx
@@ -2237,16 +2247,12 @@ func (d *RocksDB) FixUtxos(stop chan os.Signal) error {
 	return nil
 }
 
-func (d *RocksDB) storeBlockFilter(wb *grocksdb.WriteBatch, blockHash string, blockFilter string) error {
+func (d *RocksDB) storeBlockFilter(wb *grocksdb.WriteBatch, blockHash string, blockFilter []byte) error {
 	blockHashBytes, err := hex.DecodeString(blockHash)
 	if err != nil {
 		return err
 	}
-	blockFilterBytes, err := hex.DecodeString(blockFilter)
+	wb.PutCF(d.cfh[cfBlockFilter], blockHashBytes, blockFilter)
-	if err != nil {
-		return err
-	}
-	wb.PutCF(d.cfh[cfBlockFilter], blockHashBytes, blockFilterBytes)
 	return nil
 }
 

db/rocksdb_test.go

@@ -810,6 +810,7 @@ func Test_BlockFilter_GetAndStore(t *testing.T) {
 
 	blockHash := "0000000000000003d0c9722718f8ee86c2cf394f9cd458edb1c854de2a7b1a91"
 	blockFilter := "042c6340895e413d8a811fa0"
+	blockFilterBytes, _ := hex.DecodeString(blockFilter)
 
 	// Empty at the beginning
 	got, err := d.GetBlockFilter(blockHash)
@@ -823,7 +824,7 @@ func Test_BlockFilter_GetAndStore(t *testing.T) {
 
 	// Store the filter
 	wb := grocksdb.NewWriteBatch()
-	if err := d.storeBlockFilter(wb, blockHash, blockFilter); err != nil {
+	if err := d.storeBlockFilter(wb, blockHash, blockFilterBytes); err != nil {
 		t.Fatal(err)
 	}
 	if err := d.WriteBatch(wb); err != nil {