From 917840d6b2251ac99583fc155e6036195814cf42 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 May 2019 14:10:26 +0200 Subject: [PATCH] Stop indexing OP_RETURN scripts --- bchain/baseparser.go | 6 ++ bchain/coins/btc/bitcoinparser.go | 9 +++ bchain/types.go | 1 + db/rocksdb.go | 92 ++++++++++++++++--------------- db/rocksdb_test.go | 13 ++++- server/public_test.go | 4 +- tests/dbtestdata/dbtestdata.go | 9 +++ 7 files changed, 86 insertions(+), 48 deletions(-) diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 55f055e8..90d5c5a2 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -268,6 +268,12 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { return &tx, pt.Height, nil } +// IsAddrDescIndexable returns true if AddressDescriptor should be added to index +// by default all AddressDescriptors are indexable +func (p *BaseParser) IsAddrDescIndexable(addrDesc AddressDescriptor) bool { + return true +} + // DerivationBasePath is unsupported func (p *BaseParser) DerivationBasePath(xpub string) (string, error) { return "", errors.New("Not supported") diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 01d17d01..24d236ef 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -91,6 +91,15 @@ func (p *BitcoinParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) return addrDesc, nil } +// IsAddrDescIndexable returns true if AddressDescriptor should be added to index +// empty or OP_RETURN scripts are not indexed +func (p *BitcoinParser) IsAddrDescIndexable(addrDesc bchain.AddressDescriptor) bool { + if len(addrDesc) == 0 || addrDesc[0] == txscript.OP_RETURN { + return false + } + return true +} + // addressToOutputScript converts bitcoin address to ScriptPubKey func (p *BitcoinParser) addressToOutputScript(address string) ([]byte, error) { da, err := btcutil.DecodeAddress(address, p.Params) diff --git a/bchain/types.go b/bchain/types.go index 4b3c1abd..7e87fa74 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -264,6 +264,7 @@ type BlockChainParser interface { GetAddrDescFromAddress(address string) (AddressDescriptor, error) GetAddressesFromAddrDesc(addrDesc AddressDescriptor) ([]string, bool, error) GetScriptFromAddrDesc(addrDesc AddressDescriptor) ([]byte, error) + IsAddrDescIndexable(addrDesc AddressDescriptor) bool // transactions PackedTxidLen() int PackTxid(txid string) ([]byte, error) diff --git a/db/rocksdb.go b/db/rocksdb.go index 3c1e65f4..f9dfdf91 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -459,25 +459,27 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add continue } tao.AddrDesc = addrDesc - strAddrDesc := string(addrDesc) - ab, e := balances[strAddrDesc] - if !e { - ab, err = d.GetAddrDescBalance(addrDesc) - if err != nil { - return err + if d.chainParser.IsAddrDescIndexable(addrDesc) { + strAddrDesc := string(addrDesc) + balance, e := balances[strAddrDesc] + if !e { + balance, err = d.GetAddrDescBalance(addrDesc) + if err != nil { + return err + } + if balance == nil { + balance = &AddrBalance{} + } + balances[strAddrDesc] = balance + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ } - if ab == nil { - ab = &AddrBalance{} + balance.BalanceSat.Add(&balance.BalanceSat, &output.ValueSat) + counted := addToAddressesMap(addresses, strAddrDesc, btxID, int32(i)) + if !counted { + balance.Txs++ } - balances[strAddrDesc] = ab - d.cbs.balancesMiss++ - } else { - d.cbs.balancesHit++ - } - ab.BalanceSat.Add(&ab.BalanceSat, &output.ValueSat) - counted := addToAddressesMap(addresses, strAddrDesc, btxID, int32(i)) - if !counted { - ab.Txs++ } } } @@ -519,45 +521,47 @@ func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses add glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) continue } - ot := &ita.Outputs[int(input.Vout)] - if ot.Spent { + spentOutput := &ita.Outputs[int(input.Vout)] + if spentOutput.Spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } - tai.AddrDesc = ot.AddrDesc - tai.ValueSat = ot.ValueSat + tai.AddrDesc = spentOutput.AddrDesc + tai.ValueSat = spentOutput.ValueSat // mark the output as spent in tx - ot.Spent = true - if len(ot.AddrDesc) == 0 { + spentOutput.Spent = true + if len(spentOutput.AddrDesc) == 0 { if !logged { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) logged = true } continue } - strAddrDesc := string(ot.AddrDesc) - ab, e := balances[strAddrDesc] - if !e { - ab, err = d.GetAddrDescBalance(ot.AddrDesc) - if err != nil { - return err + if d.chainParser.IsAddrDescIndexable(spentOutput.AddrDesc) { + strAddrDesc := string(spentOutput.AddrDesc) + balance, e := balances[strAddrDesc] + if !e { + balance, err = d.GetAddrDescBalance(spentOutput.AddrDesc) + if err != nil { + return err + } + if balance == nil { + balance = &AddrBalance{} + } + balances[strAddrDesc] = balance + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ } - if ab == nil { - ab = &AddrBalance{} + counted := addToAddressesMap(addresses, strAddrDesc, spendingTxid, ^int32(i)) + if !counted { + balance.Txs++ } - balances[strAddrDesc] = ab - d.cbs.balancesMiss++ - } else { - d.cbs.balancesHit++ + balance.BalanceSat.Sub(&balance.BalanceSat, &spentOutput.ValueSat) + if balance.BalanceSat.Sign() < 0 { + d.resetValueSatToZero(&balance.BalanceSat, spentOutput.AddrDesc, "balance") + } + balance.SentSat.Add(&balance.SentSat, &spentOutput.ValueSat) } - counted := addToAddressesMap(addresses, strAddrDesc, spendingTxid, ^int32(i)) - if !counted { - ab.Txs++ - } - ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat) - if ab.BalanceSat.Sign() < 0 { - d.resetValueSatToZero(&ab.BalanceSat, ot.AddrDesc, "balance") - } - ab.SentSat.Add(&ab.SentSat, &ot.ValueSat) } } return nil diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index ac938968..fbe3d4d6 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -315,9 +315,10 @@ func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) { "02" + inputAddressToPubKeyHexWithLength(dbtestdata.Addr3, t, d) + bigintToHex(dbtestdata.SatB1T2A3) + inputAddressToPubKeyHexWithLength(dbtestdata.Addr2, t, d) + bigintToHex(dbtestdata.SatB1T1A2) + - "02" + + "03" + spentAddressToPubKeyHexWithLength(dbtestdata.Addr6, t, d) + bigintToHex(dbtestdata.SatB2T1A6) + - addressToPubKeyHexWithLength(dbtestdata.Addr7, t, d) + bigintToHex(dbtestdata.SatB2T1A7), + addressToPubKeyHexWithLength(dbtestdata.Addr7, t, d) + bigintToHex(dbtestdata.SatB2T1A7) + + hex.EncodeToString([]byte{byte(len(dbtestdata.TxidB2T1Output3OpReturn))}) + dbtestdata.TxidB2T1Output3OpReturn + bigintToHex(dbtestdata.SatZero), nil, }, { @@ -624,6 +625,14 @@ func TestRocksDB_Index_BitcoinType(t *testing.T) { Spent: false, ValueSat: *dbtestdata.SatB2T1A7, }, + { + AddrDesc: func() []byte { + b, _ := hex.DecodeString(dbtestdata.TxidB2T1Output3OpReturn) + return b + }(), + Spent: false, + ValueSat: *dbtestdata.SatZero, + }, }, } if !reflect.DeepEqual(ta, taw) { diff --git a/server/public_test.go b/server/public_test.go index 172bb17c..c1069668 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -480,7 +480,7 @@ func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ - `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw","balance":"0","totalReceived":"1234567890123","totalSent":"1234567890123","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":2,"transactions":[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","n":0,"addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"],"value":"1234567890123"},{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","vout":1,"n":1,"addresses":["mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"],"value":"12345"}],"vout":[{"value":"317283951061","n":0,"spent":true,"hex":"76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac","addresses":["mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX"]},{"value":"917283951061","n":1,"hex":"76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac","addresses":["mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"]}],"blockHash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockHeight":225494,"confirmations":1,"blockTime":22549400000,"value":"1234567902122","valueIn":"1234567902468","fees":"346"},{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vin":[],"vout":[{"value":"1234567890123","n":0,"spent":true,"hex":"76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac","addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"]},{"value":"1","n":1,"spent":true,"hex":"a91452724c5178682f70e0ba31c6ec0633755a3b41d987","addresses":["2MzmAKayJmja784jyHvRUW1bXPget1csRRG"]},{"value":"9876","n":2,"spent":true,"hex":"a914e921fc4912a315078f370d959f2c4f7b6d2a683c87","addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"]}],"blockHash":"0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997","blockHeight":225493,"confirmations":2,"blockTime":22549300001,"value":"1234567900000","valueIn":"0","fees":"0"}]}`, + `{"page":1,"totalPages":1,"itemsOnPage":1000,"address":"mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw","balance":"0","totalReceived":"1234567890123","totalSent":"1234567890123","unconfirmedBalance":"0","unconfirmedTxs":0,"txs":2,"transactions":[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","n":0,"addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"],"value":"1234567890123"},{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","vout":1,"n":1,"addresses":["mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"],"value":"12345"}],"vout":[{"value":"317283951061","n":0,"spent":true,"hex":"76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac","addresses":["mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX"]},{"value":"917283951061","n":1,"hex":"76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac","addresses":["mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"]},{"value":"0","n":2,"hex":"6a072020f1686f6a20","addresses":["OP_RETURN 2020f1686f6a20"]}],"blockHash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockHeight":225494,"confirmations":1,"blockTime":22549400000,"value":"1234567902122","valueIn":"1234567902468","fees":"346"},{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vin":[],"vout":[{"value":"1234567890123","n":0,"spent":true,"hex":"76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac","addresses":["mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"]},{"value":"1","n":1,"spent":true,"hex":"a91452724c5178682f70e0ba31c6ec0633755a3b41d987","addresses":["2MzmAKayJmja784jyHvRUW1bXPget1csRRG"]},{"value":"9876","n":2,"spent":true,"hex":"a914e921fc4912a315078f370d959f2c4f7b6d2a683c87","addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"]}],"blockHash":"0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997","blockHeight":225493,"confirmations":2,"blockTime":22549300001,"value":"1234567900000","valueIn":"0","fees":"0"}]}`, }, }, { @@ -735,7 +735,7 @@ func socketioTests_BitcoinType(t *testing.T, ts *httptest.Server) { "to": 5, }, }}, - want: `{"result":{"totalCount":2,"items":[{"addresses":{"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz":{"inputIndexes":[1],"outputIndexes":[]}},"satoshis":-12345,"confirmations":1,"tx":{"hex":"","height":225494,"blockTimestamp":22549400000,"version":0,"hash":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","inputs":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","outputIndex":0,"script":"","sequence":0,"address":"mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw","satoshis":1234567890123},{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","outputIndex":1,"script":"","sequence":0,"address":"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz","satoshis":12345}],"inputSatoshis":1234567902468,"outputs":[{"satoshis":317283951061,"script":"76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac","address":"mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX"},{"satoshis":917283951061,"script":"76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac","address":"mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"}],"outputSatoshis":1234567902122,"feeSatoshis":346}},{"addresses":{"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz":{"inputIndexes":[],"outputIndexes":[1]}},"satoshis":12345,"confirmations":2,"tx":{"hex":"","height":225493,"blockTimestamp":22549300000,"version":0,"hash":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","inputs":[],"outputs":[{"satoshis":100000000,"script":"76a914010d39800f86122416e28f485029acf77507169288ac","address":"mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti"},{"satoshis":12345,"script":"76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac","address":"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"}],"outputSatoshis":100012345}}]}}`, + want: `{"result":{"totalCount":2,"items":[{"addresses":{"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz":{"inputIndexes":[1],"outputIndexes":[]}},"satoshis":-12345,"confirmations":1,"tx":{"hex":"","height":225494,"blockTimestamp":22549400000,"version":0,"hash":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","inputs":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","outputIndex":0,"script":"","sequence":0,"address":"mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw","satoshis":1234567890123},{"txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","outputIndex":1,"script":"","sequence":0,"address":"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz","satoshis":12345}],"inputSatoshis":1234567902468,"outputs":[{"satoshis":317283951061,"script":"76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac","address":"mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX"},{"satoshis":917283951061,"script":"76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac","address":"mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"},{"satoshis":0,"script":"6a072020f1686f6a20","address":"OP_RETURN 2020f1686f6a20"}],"outputSatoshis":1234567902122,"feeSatoshis":346}},{"addresses":{"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz":{"inputIndexes":[],"outputIndexes":[1]}},"satoshis":12345,"confirmations":2,"tx":{"hex":"","height":225493,"blockTimestamp":22549300000,"version":0,"hash":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","inputs":[],"outputs":[{"satoshis":100000000,"script":"76a914010d39800f86122416e28f485029acf77507169288ac","address":"mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti"},{"satoshis":12345,"script":"76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac","address":"mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz"}],"outputSatoshis":100012345}}]}}`, }, { name: "getBlockHeader", diff --git a/tests/dbtestdata/dbtestdata.go b/tests/dbtestdata/dbtestdata.go index ab2595a3..4a12d85d 100644 --- a/tests/dbtestdata/dbtestdata.go +++ b/tests/dbtestdata/dbtestdata.go @@ -29,6 +29,8 @@ const ( Addr8 = "2N6utyMZfPNUb1Bk8oz7p2JqJrXkq83gegu" // a91495e9fbe306449c991d314afe3c3567d5bf78efd287, xpub m/49'/1'/33'/1/3 Addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac AddrA = "mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj" // 76a914d03c0d863d189b23b061a95ad32940b65837609f88ac + + TxidB2T1Output3OpReturn = "6a072020f1686f6a20" ) // Amounts in satoshis @@ -166,6 +168,13 @@ func GetTestBitcoinTypeBlock2(parser bchain.BlockChainParser) *bchain.Block { }, ValueSat: *SatB2T1A7, }, + { + N: 2, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: TxidB2T1Output3OpReturn, // OP_RETURN script + }, + ValueSat: *SatZero, + }, }, Blocktime: 22549400000, Time: 22549400000,