diff --git a/bchain/golomb.go b/bchain/golomb.go new file mode 100644 index 00000000..f5e89860 --- /dev/null +++ b/bchain/golomb.go @@ -0,0 +1,102 @@ +package bchain + +import ( + "encoding/hex" + + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/martinboehm/btcutil/gcs" +) + +type FilterScriptsType int + +const ( + FilterScriptsInvalid = FilterScriptsType(iota) + FilterScriptsAll + FilterScriptsTaproot +) + +// GolombFilter is computing golomb filter of address descriptors +type GolombFilter struct { + Enabled bool + p uint8 + key string + filterScripts string + filterScriptsType FilterScriptsType + filterData [][]byte + uniqueData map[string]struct{} +} + +// NewGolombFilter initializes the GolombFilter handler +func NewGolombFilter(p uint8, filterScripts string, key string) (*GolombFilter, error) { + if p == 0 { + return &GolombFilter{Enabled: false}, nil + } + gf := GolombFilter{ + Enabled: true, + p: p, + key: key, + filterScripts: filterScripts, + filterScriptsType: filterScriptsToScriptsType(filterScripts), + filterData: make([][]byte, 0), + uniqueData: make(map[string]struct{}), + } + // only taproot and all is supported + if gf.filterScriptsType == FilterScriptsInvalid { + return nil, errors.Errorf("Invalid/unsupported filterScripts parameter %s", filterScripts) + } + return &gf, nil +} + +// AddAddrDesc adds taproot address descriptor to the data for the filter +func (f *GolombFilter) AddAddrDesc(ad AddressDescriptor) { + if f.filterScriptsType == FilterScriptsTaproot && !ad.IsTaproot() { + return + } + if len(ad) == 0 { + return + } + s := string(ad) + if _, found := f.uniqueData[s]; !found { + f.filterData = append(f.filterData, ad) + f.uniqueData[s] = struct{}{} + } +} + +// Compute computes golomb filter from the data +func (f *GolombFilter) Compute() []byte { + m := uint64(1 << uint64(f.p)) + + if len(f.filterData) == 0 { + return nil + } + + b, _ := hex.DecodeString(f.key) + if len(b) < gcs.KeySize { + return nil + } + + filter, err := gcs.BuildGCSFilter(f.p, m, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), f.filterData) + if err != nil { + glog.Error("Cannot create golomb filter for ", f.key, ", ", err) + return nil + } + + fb, err := filter.NBytes() + if err != nil { + glog.Error("Error getting NBytes from golomb filter for ", f.key, ", ", err) + return nil + } + + return fb +} + +func filterScriptsToScriptsType(filterScripts string) FilterScriptsType { + switch filterScripts { + case "": + return FilterScriptsAll + case "taproot": + return FilterScriptsTaproot + } + return FilterScriptsInvalid +} diff --git a/bchain/golomb_test.go b/bchain/golomb_test.go new file mode 100644 index 00000000..eba1ab41 --- /dev/null +++ b/bchain/golomb_test.go @@ -0,0 +1,117 @@ +// //go:build unittest + +package bchain + +import ( + "encoding/hex" + "testing" +) + +func TestGolombFilter(t *testing.T) { + tests := []struct { + name string + p uint8 + filterScripts string + key string + addressDescriptors [][]byte + wantError bool + wantEnabled bool + want string + }{ + { + name: "taproot", + p: 20, + filterScripts: "taproot", + key: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2", + addressDescriptors: [][]byte{ + // bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5 + hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"), + // bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav + hexToBytes("5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f"), + // 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ + hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"), + }, + wantEnabled: true, + wantError: false, + want: "0235dddcce5d60", + }, + { + name: "taproot p=21", + p: 21, + filterScripts: "taproot", + key: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2", + addressDescriptors: [][]byte{ + // bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5 + hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"), + // bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav + hexToBytes("5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f"), + // 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ + hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"), + }, + wantEnabled: true, + wantError: false, + want: "0235ddda672eb0", + }, + { + name: "all", + p: 20, + filterScripts: "", + key: "86336c62a63f509a278624e3f400cdd50838d035a44e0af8a7d6d133c04cc2d2", + addressDescriptors: [][]byte{ + // bc1pgeqrcq5capal83ypxczmypjdhk4d9wwcea4k66c7ghe07p2qt97sqh8sy5 + hexToBytes("512046403c0298e87bf3c4813605b2064dbdaad2b9d8cf6b6d6b1e45f2ff0540597d"), + // bc1p7en40zu9hmf9d3luh8evmfyg655pu5k2gtna6j7zr623f9tz7z0stfnwav + hexToBytes("5120f667578b85bed256c7fcb9f2cda488d5281e52ca42e7dd4bc21e95149562f09f"), + // 39ECUF8YaFRX7XfttfAiLa5ir43bsrQUZJ + hexToBytes("a91452ae9441d9920d9eb4a3c0a877ca8d8de547ce6587"), + }, + wantEnabled: true, + wantError: false, + want: "0350ccc61ac611976c80", + }, + { + name: "not supported filter", + p: 20, + filterScripts: "notsupported", + wantEnabled: false, + wantError: true, + want: "", + }, + { + name: "not enabled", + p: 0, + filterScripts: "", + wantEnabled: false, + wantError: false, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gf, err := NewGolombFilter(tt.p, tt.filterScripts, tt.key) + if err != nil && !tt.wantError { + t.Errorf("TestGolombFilter.NewGolombFilter() got unexpected error '%v'", err) + return + } + if err == nil && tt.wantError { + t.Errorf("TestGolombFilter.NewGolombFilter() wanted error, got none") + return + } + if gf == nil && tt.wantError { + return + } + if gf.Enabled != tt.wantEnabled { + t.Errorf("TestGolombFilter.NewGolombFilter() got gf.Enabled %v, want %v", gf.Enabled, tt.wantEnabled) + return + } + for _, ad := range tt.addressDescriptors { + gf.AddAddrDesc(ad) + } + f := gf.Compute() + got := hex.EncodeToString(f) + if got != tt.want { + t.Errorf("TestGolombFilter Compute() got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index cce99d8b..5a63e756 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -2,13 +2,11 @@ package bchain import ( "encoding/hex" - "errors" - "fmt" "math/big" "time" "github.com/golang/glog" - "github.com/martinboehm/btcutil/gcs" + "github.com/juju/errors" ) type chanInputPayload struct { @@ -16,14 +14,6 @@ type chanInputPayload struct { index int } -type filterScriptsType int - -const ( - filterScriptsInvalid = filterScriptsType(iota) - filterScriptsAll - filterScriptsTaproot -) - // MempoolBitcoinType is mempool handle. type MempoolBitcoinType struct { BaseMempool @@ -31,19 +21,12 @@ type MempoolBitcoinType struct { chanAddrIndex chan txidio AddrDescForOutpoint AddrDescForOutpointFunc golombFilterP uint8 - golombFilterM uint64 - filterScripts filterScriptsType + filterScripts string } // NewMempoolBitcoinType creates new mempool handler. // For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8, filterScripts string) *MempoolBitcoinType { - filterScriptsType := filterScriptsToScriptsType(filterScripts) - if filterScriptsType == filterScriptsInvalid { - glog.Error("Invalid filterScripts ", filterScripts, ", switching off golomb filter") - golombFilterP = 0 - } - golombFilterM := uint64(1 << golombFilterP) m := &MempoolBitcoinType{ BaseMempool: BaseMempool{ chain: chain, @@ -53,8 +36,7 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golomb chanTxid: make(chan string, 1), chanAddrIndex: make(chan txidio, 1), golombFilterP: golombFilterP, - golombFilterM: golombFilterM, - filterScripts: filterScriptsType, + filterScripts: filterScripts, } for i := 0; i < workers; i++ { go func(i int) { @@ -81,16 +63,6 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golomb return m } -func filterScriptsToScriptsType(filterScripts string) filterScriptsType { - switch filterScripts { - case "": - return filterScriptsAll - case "taproot": - return filterScriptsTaproot - } - return filterScriptsInvalid -} - func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrIndex { var addrDesc AddressDescriptor var value *big.Int @@ -126,49 +98,20 @@ func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrInd } func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx) string { - uniqueScripts := make(map[string]struct{}) - filterData := make([][]byte, 0) - - handleAddrDesc := func(ad AddressDescriptor) { - if m.filterScripts == filterScriptsAll || (m.filterScripts == filterScriptsTaproot && ad.IsTaproot()) { - if len(ad) == 0 { - return - } - s := string(ad) - if _, found := uniqueScripts[s]; !found { - filterData = append(filterData, ad) - uniqueScripts[s] = struct{}{} - } - } + gf, _ := NewGolombFilter(m.golombFilterP, m.filterScripts, mtx.Txid) + if gf == nil || !gf.Enabled { + return "" } - for _, vin := range mtx.Vin { - handleAddrDesc(vin.AddrDesc) + gf.AddAddrDesc(vin.AddrDesc) } for _, vout := range mtx.Vout { b, err := hex.DecodeString(vout.ScriptPubKey.Hex) if err == nil { - handleAddrDesc(b) + gf.AddAddrDesc(b) } } - - if len(filterData) == 0 { - return "" - } - b, _ := hex.DecodeString(mtx.Txid) - if len(b) < gcs.KeySize { - return "" - } - filter, err := gcs.BuildGCSFilter(m.golombFilterP, m.golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), filterData) - if err != nil { - glog.Error("Cannot create golomb filter for ", mtx.Txid, ", ", err) - return "" - } - fb, err := filter.NBytes() - if err != nil { - glog.Error("Error getting NBytes from golomb filter for ", mtx.Txid, ", ", err) - return "" - } + fb := gf.Compute() return hex.EncodeToString(fb) } @@ -295,8 +238,8 @@ func (m *MempoolBitcoinType) Resync() (int, error) { // GetTxidFilterEntries returns all mempool entries with golomb filter from func (m *MempoolBitcoinType) GetTxidFilterEntries(filterScripts string, fromTimestamp uint32) (MempoolTxidFilterEntries, error) { - if m.filterScripts != filterScriptsToScriptsType(filterScripts) { - return MempoolTxidFilterEntries{}, errors.New(fmt.Sprint("Unsupported script filter ", filterScripts)) + if m.filterScripts != filterScripts { + return MempoolTxidFilterEntries{}, errors.Errorf("Unsupported script filter %s", filterScripts) } m.mux.Lock() entries := make(map[string]string) diff --git a/bchain/mempool_bitcoin_type_test.go b/bchain/mempool_bitcoin_type_test.go index 194f5e19..9201be4c 100644 --- a/bchain/mempool_bitcoin_type_test.go +++ b/bchain/mempool_bitcoin_type_test.go @@ -16,9 +16,9 @@ func TestMempoolBitcoinType_computeGolombFilter_taproot(t *testing.T) { randomScript := hexToBytes("a914ff074800343a81ada8fe86c2d5d5a0e55b93dd7a87") m := &MempoolBitcoinType{ golombFilterP: 20, - golombFilterM: uint64(1 << 20), - filterScripts: filterScriptsTaproot, + filterScripts: "taproot", } + golombFilterM := uint64(1 << uint64(m.golombFilterP)) tests := []struct { name string mtx MempoolTx @@ -200,7 +200,7 @@ func TestMempoolBitcoinType_computeGolombFilter_taproot(t *testing.T) { } if got != "" { // build the filter from computed value - filter, err := gcs.FromNBytes(m.golombFilterP, m.golombFilterM, hexToBytes(got)) + filter, err := gcs.FromNBytes(m.golombFilterP, golombFilterM, hexToBytes(got)) if err != nil { t.Errorf("gcs.BuildGCSFilter() unexpected error %v", err) } diff --git a/db/blockfilter.go b/db/blockfilter.go deleted file mode 100644 index f6d73ea7..00000000 --- a/db/blockfilter.go +++ /dev/null @@ -1,68 +0,0 @@ -package db - -import ( - "encoding/hex" - - "github.com/golang/glog" - "github.com/martinboehm/btcutil/gcs" - "github.com/trezor/blockbook/bchain" -) - -func computeBlockFilter(allAddrDesc [][]byte, blockHash string, taprootOnly bool) string { - // TODO: take these from config - how to access it? From BitcoinRPC? - // TODO: these things should probably be an argument to this function, - // so it is better testable - golombFilterP := uint8(20) - golombFilterM := uint64(1 << golombFilterP) - - // TODO: code below is almost a copy-paste from computeGolombFilter, - // it might be possible to refactor it into a common function, e.g. - // computeGolomb(allAddrDescriptors, P, M, taprootOnly, hashIdentifier) -> filterData - // but where to put it? - - uniqueScripts := make(map[string]struct{}) - filterData := make([][]byte, 0) - - handleAddrDesc := func(ad bchain.AddressDescriptor) { - if taprootOnly && !ad.IsTaproot() { - return - } - if len(ad) == 0 { - return - } - s := string(ad) - if _, found := uniqueScripts[s]; !found { - filterData = append(filterData, ad) - uniqueScripts[s] = struct{}{} - } - } - - for _, ad := range allAddrDesc { - handleAddrDesc(ad) - } - - if len(filterData) == 0 { - return "" - } - - b, _ := hex.DecodeString(blockHash) - if len(b) < gcs.KeySize { - return "" - } - - filter, err := gcs.BuildGCSFilter(golombFilterP, golombFilterM, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), filterData) - if err != nil { - glog.Error("Cannot create golomb filter for ", blockHash, ", ", err) - return "" - } - - fb, err := filter.NBytes() - if err != nil { - glog.Error("Error getting NBytes from golomb filter for ", blockHash, ", ", err) - return "" - } - - // TODO: maybe not returning string but []byte, when we are saving it - // as []byte anyway? - return hex.EncodeToString(fb) -} diff --git a/db/blockfilter_test.go b/db/blockfilter_test.go deleted file mode 100644 index ae90245c..00000000 --- a/db/blockfilter_test.go +++ /dev/null @@ -1,98 +0,0 @@ -//go:build unittest - -package db - -import ( - "math/big" - "testing" - - "github.com/trezor/blockbook/tests/dbtestdata" -) - -func TestComputeBlockFilter(t *testing.T) { - // TODO: add more (vectorized) tests, with taproot txs - // - both taprootOnly=true and taprootOnly=false - // - check that decoding with different P does not work - allAddrDesc := getallAddrDesc() - blockHash := "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" - taprootOnly := false - got := computeBlockFilter(allAddrDesc, blockHash, taprootOnly) - want := "0847a3118f0a689307a375c45c1b02379119579910ee80" - if got != want { - t.Errorf("computeBlockFilter() failed, expected: %s, got: %s", want, got) - } -} - -func getallAddrDesc() [][]byte { - allAddrDesc := make([][]byte, 0) - parser := bitcoinTestnetParser() - - // TODO: this data is copied exactly, make it common and reuse it - ta := &TxAddresses{ - Height: 12345, - VSize: 321, - Inputs: []TxInput{ - { - AddrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), - ValueSat: *big.NewInt(9011000000), - Txid: "c50c7ce2f5670fd52de738288299bd854a85ef1bb304f62f35ced1bd49a8a810", - Vout: 0, - }, - { - AddrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), - ValueSat: *big.NewInt(8011000000), - Txid: "e96672c7fcc8da131427fcea7e841028614813496a56c11e8a6185c16861c495", - Vout: 1, - }, - { - AddrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), - ValueSat: *big.NewInt(7011000000), - Txid: "ed308c72f9804dfeefdbb483ef8fd1e638180ad81d6b33f4b58d36d19162fa6d", - Vout: 134, - }, - }, - Outputs: []TxOutput{ - { - AddrDesc: addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), - ValueSat: *big.NewInt(5011000000), - Spent: true, - SpentTxid: dbtestdata.TxidB1T1, - SpentIndex: 0, - SpentHeight: 432112345, - }, - { - AddrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), - ValueSat: *big.NewInt(6011000000), - }, - { - AddrDesc: addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), - ValueSat: *big.NewInt(7011000000), - Spent: true, - SpentTxid: dbtestdata.TxidB1T2, - SpentIndex: 14231, - SpentHeight: 555555, - }, - { - AddrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), - ValueSat: *big.NewInt(999900000), - }, - { - AddrDesc: addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), - ValueSat: *big.NewInt(5000000000), - Spent: true, - SpentTxid: dbtestdata.TxidB2T1, - SpentIndex: 674541, - SpentHeight: 6666666, - }, - }, - } - - for _, input := range ta.Inputs { - allAddrDesc = append(allAddrDesc, input.AddrDesc) - } - for _, output := range ta.Outputs { - allAddrDesc = append(allAddrDesc, output.AddrDesc) - } - - return allAddrDesc -} diff --git a/db/bulkconnect.go b/db/bulkconnect.go index e0adc607..07c7988f 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -27,7 +27,7 @@ type BulkConnect struct { bulkAddressesCount int ethBlockTxs []ethBlockTx txAddressesMap map[string]*TxAddresses - blockFilters map[string]string + blockFilters map[string][]byte balances map[string]*AddrBalance addressContracts map[string]*AddrContracts height uint32 @@ -52,6 +52,7 @@ func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { txAddressesMap: make(map[string]*TxAddresses), balances: make(map[string]*AddrBalance), addressContracts: make(map[string]*AddrContracts), + blockFilters: make(map[string][]byte), } if err := d.SetInconsistentState(true); err != nil { return nil, err @@ -178,14 +179,20 @@ func (b *BulkConnect) storeBulkBlockFilters(wb *grocksdb.WriteBatch) error { return err } } - b.blockFilters = make(map[string]string) + b.blockFilters = make(map[string][]byte) return nil } func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs bool) error { addresses := make(addressesMap) - allBlockAddrDesc := make([][]byte, 0) - if err := b.d.processAddressesBitcoinType(block, addresses, b.txAddressesMap, b.balances, &allBlockAddrDesc); err != nil { + gf, err := bchain.NewGolombFilter(b.d.is.BlockGolombFilterP, b.d.is.BlockFilterScripts, block.BlockHeader.Hash) + if err != nil { + glog.Error("connectBlockBitcoinType golomb filter error ", err) + gf = nil + } else if gf != nil && !gf.Enabled { + gf = nil + } + if err := b.d.processAddressesBitcoinType(block, addresses, b.txAddressesMap, b.balances, gf); err != nil { return err } var storeAddressesChan, storeBalancesChan chan error @@ -212,11 +219,9 @@ func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs addresses: addresses, }) b.bulkAddressesCount += len(addresses) - if b.blockFilters == nil { - b.blockFilters = make(map[string]string) // TODO: where to put this? + if gf != nil { + b.blockFilters[block.BlockHeader.Hash] = gf.Compute() } - taprootOnly := true // TODO: take from config - b.blockFilters[block.BlockHeader.Hash] = computeBlockFilter(allBlockAddrDesc, block.BlockHeader.Hash, taprootOnly) // open WriteBatch only if going to write if sa || b.bulkAddressesCount > maxBulkAddresses || storeBlockTxs || len(b.blockFilters) > maxBlockFilters { start := time.Now() diff --git a/db/rocksdb.go b/db/rocksdb.go index 52f3e126..06743c8a 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -348,9 +348,15 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error { addresses := make(addressesMap) if chainType == bchain.ChainBitcoinType { txAddressesMap := make(map[string]*TxAddresses) - allBlockAddrDesc := make([][]byte, 0) balances := make(map[string]*AddrBalance) - if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances, &allBlockAddrDesc); err != nil { + gf, err := bchain.NewGolombFilter(d.is.BlockGolombFilterP, d.is.BlockFilterScripts, block.BlockHeader.Hash) + if err != nil { + glog.Error("ConnectBlock golomb filter error ", err) + gf = nil + } else if gf != nil && !gf.Enabled { + gf = nil + } + if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances, gf); err != nil { return err } if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { @@ -362,10 +368,11 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error { if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { return err } - taprootOnly := true // TODO: take from config - blockFilter := computeBlockFilter(allBlockAddrDesc, block.BlockHeader.Hash, taprootOnly) - if err := d.storeBlockFilter(wb, block.BlockHeader.Hash, blockFilter); err != nil { - return err + if gf != nil { + blockFilter := gf.Compute() + if err := d.storeBlockFilter(wb, block.BlockHeader.Hash, blockFilter); err != nil { + return err + } } } else if chainType == bchain.ChainEthereumType { addressContracts := make(map[string]*AddrContracts) @@ -597,8 +604,7 @@ func (d *RocksDB) GetAndResetConnectBlockStats() string { return s } -// TODO: maybe return allBlockAddrDesc from this function instead of taking it as argument -func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses addressesMap, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance, allBlockAddrDesc *[][]byte) error { +func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses addressesMap, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance, gf *bchain.GolombFilter) error { blockTxIDs := make([][]byte, len(block.Txs)) blockTxAddresses := make([]*TxAddresses, len(block.Txs)) // first process all outputs so that inputs can refer to txs in this block @@ -636,7 +642,9 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add } continue } - *allBlockAddrDesc = append(*allBlockAddrDesc, addrDesc) // new addrDesc + if gf != nil { + gf.AddAddrDesc(addrDesc) + } tao.AddrDesc = addrDesc if d.chainParser.IsAddrDescIndexable(addrDesc) { strAddrDesc := string(addrDesc) @@ -711,7 +719,9 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add if spentOutput.Spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } - *allBlockAddrDesc = append(*allBlockAddrDesc, spentOutput.AddrDesc) // new addrDesc + if gf != nil { + gf.AddAddrDesc(spentOutput.AddrDesc) + } tai.AddrDesc = spentOutput.AddrDesc tai.ValueSat = spentOutput.ValueSat // mark the output as spent in tx @@ -2237,16 +2247,12 @@ func (d *RocksDB) FixUtxos(stop chan os.Signal) error { return nil } -func (d *RocksDB) storeBlockFilter(wb *grocksdb.WriteBatch, blockHash string, blockFilter string) error { +func (d *RocksDB) storeBlockFilter(wb *grocksdb.WriteBatch, blockHash string, blockFilter []byte) error { blockHashBytes, err := hex.DecodeString(blockHash) if err != nil { return err } - blockFilterBytes, err := hex.DecodeString(blockFilter) - if err != nil { - return err - } - wb.PutCF(d.cfh[cfBlockFilter], blockHashBytes, blockFilterBytes) + wb.PutCF(d.cfh[cfBlockFilter], blockHashBytes, blockFilter) return nil } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 2e4330c0..1204b0c3 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -810,6 +810,7 @@ func Test_BlockFilter_GetAndStore(t *testing.T) { blockHash := "0000000000000003d0c9722718f8ee86c2cf394f9cd458edb1c854de2a7b1a91" blockFilter := "042c6340895e413d8a811fa0" + blockFilterBytes, _ := hex.DecodeString(blockFilter) // Empty at the beginning got, err := d.GetBlockFilter(blockHash) @@ -823,7 +824,7 @@ func Test_BlockFilter_GetAndStore(t *testing.T) { // Store the filter wb := grocksdb.NewWriteBatch() - if err := d.storeBlockFilter(wb, blockHash, blockFilter); err != nil { + if err := d.storeBlockFilter(wb, blockHash, blockFilterBytes); err != nil { t.Fatal(err) } if err := d.WriteBatch(wb); err != nil {