From 8b20c3813696e4fc39ca4b19deb0cd8ad80b0fed Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 28 May 2018 14:57:44 +0200 Subject: [PATCH] Put parameter BlockAddressesToKeep to coin configuration json --- bchain/baseparser.go | 5 +- bchain/coins/bch/bcashparser.go | 11 ++-- bchain/coins/bch/bcashparser_test.go | 7 +-- bchain/coins/bch/bcashrpc.go | 2 +- bchain/coins/btc/bitcoinparser.go | 5 +- bchain/coins/btc/bitcoinparser_test.go | 12 ++--- bchain/coins/btc/bitcoinrpc.go | 69 ++++++++++++++------------ bchain/coins/zec/zcashparser.go | 10 +++- bchain/coins/zec/zcashparser_test.go | 9 ++-- bchain/coins/zec/zcashrpc.go | 2 +- configs/bch-testnet.json | 3 +- configs/bch.json | 3 +- configs/btc-testnet.json | 3 +- configs/btc.json | 3 +- configs/zec.json | 3 +- db/rocksdb_test.go | 12 ++--- 16 files changed, 90 insertions(+), 69 deletions(-) diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 14b7dfed..16ec5fab 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -11,7 +11,8 @@ import ( // BaseParser implements data parsing/handling functionality base for all other parsers type BaseParser struct { - AddressFactory func(string) (Address, error) + AddressFactory func(string) (Address, error) + BlockAddressesToKeep int } // AddressToOutputScript converts address to ScriptPubKey - currently not implemented @@ -62,7 +63,7 @@ func (p *BaseParser) PackedTxidLen() int { // KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column func (p *BaseParser) KeepBlockAddresses() int { - return 100 + return p.BlockAddressesToKeep } // PackTxid packs txid to byte array diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index c2790c9e..f854169c 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -20,11 +20,14 @@ type BCashParser struct { } // NewBCashParser returns new BCashParser instance -func NewBCashParser(params *chaincfg.Params) *BCashParser { +func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) *BCashParser { return &BCashParser{ - &btc.BitcoinParser{ - &bchain.BaseParser{AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, params) }}, - params, + BitcoinParser: &btc.BitcoinParser{ + BaseParser: &bchain.BaseParser{ + AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, params) }, + BlockAddressesToKeep: c.BlockAddressesToKeep, + }, + Params: params, }, } } diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index a4c51796..26cdc1d9 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -2,6 +2,7 @@ package bch import ( "blockbook/bchain" + "blockbook/bchain/coins/btc" "bytes" "encoding/hex" "reflect" @@ -113,7 +114,7 @@ func TestBcashAddressInSlice(t *testing.T) { } func TestAddressToOutputScript(t *testing.T) { - parser := NewBCashParser(GetChainParams("test")) + parser := NewBCashParser(GetChainParams("test"), &btc.Configuration{}) want, err := hex.DecodeString("76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac") if err != nil { panic(err) @@ -247,7 +248,7 @@ func Test_UnpackTx(t *testing.T) { name: "btc-1", args: args{ packedTx: testTxPacked1, - parser: NewBCashParser(GetChainParams("main")), + parser: NewBCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: &testTx1, want1: 123456, @@ -257,7 +258,7 @@ func Test_UnpackTx(t *testing.T) { name: "testnet-1", args: args{ packedTx: testTxPacked2, - parser: NewBCashParser(GetChainParams("test")), + parser: NewBCashParser(GetChainParams("test"), &btc.Configuration{}), }, want: &testTx2, want1: 510234, diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 8ccbe993..e3cd3b49 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -41,7 +41,7 @@ func (b *BCashRPC) Initialize() error { params := GetChainParams(chainName) // always create parser - b.Parser = NewBCashParser(params) + b.Parser = NewBCashParser(params, b.ChainConfig) // parameters for getInfo request if params.Net == bchutil.MainnetMagic { diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 33a5b35e..442bcfa5 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -21,10 +21,11 @@ type BitcoinParser struct { } // NewBitcoinParser returns new BitcoinParser instance -func NewBitcoinParser(params *chaincfg.Params) *BitcoinParser { +func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser { return &BitcoinParser{ &bchain.BaseParser{ - AddressFactory: bchain.NewBaseAddress, + AddressFactory: bchain.NewBaseAddress, + BlockAddressesToKeep: c.BlockAddressesToKeep, }, params, } diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 262cb85a..6c3606d4 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -42,7 +42,7 @@ func TestAddressToOutputScript(t *testing.T) { wantErr: false, }, } - parser := NewBitcoinParser(GetChainParams("main")) + parser := NewBitcoinParser(GetChainParams("main"), &Configuration{}) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -94,7 +94,7 @@ func TestOutputScriptToAddresses(t *testing.T) { wantErr: false, }, } - parser := NewBitcoinParser(GetChainParams("main")) + parser := NewBitcoinParser(GetChainParams("main"), &Configuration{}) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b, _ := hex.DecodeString(tt.args.script) @@ -224,7 +224,7 @@ func Test_PackTx(t *testing.T) { tx: testTx1, height: 123456, blockTime: 1519053802, - parser: NewBitcoinParser(GetChainParams("main")), + parser: NewBitcoinParser(GetChainParams("main"), &Configuration{}), }, want: testTxPacked1, wantErr: false, @@ -235,7 +235,7 @@ func Test_PackTx(t *testing.T) { tx: testTx2, height: 510234, blockTime: 1235678901, - parser: NewBitcoinParser(GetChainParams("test")), + parser: NewBitcoinParser(GetChainParams("test"), &Configuration{}), }, want: testTxPacked2, wantErr: false, @@ -272,7 +272,7 @@ func Test_UnpackTx(t *testing.T) { name: "btc-1", args: args{ packedTx: testTxPacked1, - parser: NewBitcoinParser(GetChainParams("main")), + parser: NewBitcoinParser(GetChainParams("main"), &Configuration{}), }, want: &testTx1, want1: 123456, @@ -282,7 +282,7 @@ func Test_UnpackTx(t *testing.T) { name: "testnet-1", args: args{ packedTx: testTxPacked2, - parser: NewBitcoinParser(GetChainParams("test")), + parser: NewBitcoinParser(GetChainParams("test"), &Configuration{}), }, want: &testTx2, want1: 510234, diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index 75617c95..f28cbe08 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -19,39 +19,43 @@ import ( // BitcoinRPC is an interface to JSON-RPC bitcoind service. type BitcoinRPC struct { - client http.Client - rpcURL string - user string - password string - Parser bchain.BlockChainParser - Testnet bool - Network string - Mempool *bchain.UTXOMempool - ParseBlocks bool - zeroMQBinding string - pushHandler func(bchain.NotificationType) - mq *bchain.MQ - Subversion string + client http.Client + rpcURL string + user string + password string + Parser bchain.BlockChainParser + Testnet bool + Network string + Mempool *bchain.UTXOMempool + ParseBlocks bool + pushHandler func(bchain.NotificationType) + mq *bchain.MQ + ChainConfig *Configuration } -type configuration struct { - RPCURL string `json:"rpcURL"` - RPCUser string `json:"rpcUser"` - RPCPass string `json:"rpcPass"` - RPCTimeout int `json:"rpcTimeout"` - Parse bool `json:"parse"` - ZeroMQBinding string `json:"zeroMQBinding"` - Subversion string `json:"subversion"` +type Configuration struct { + RPCURL string `json:"rpcURL"` + RPCUser string `json:"rpcUser"` + RPCPass string `json:"rpcPass"` + RPCTimeout int `json:"rpcTimeout"` + Parse bool `json:"parse"` + ZeroMQBinding string `json:"zeroMQBinding"` + Subversion string `json:"subversion"` + BlockAddressesToKeep int `json:"blockAddressesToKeep"` } // NewBitcoinRPC returns new BitcoinRPC instance. func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) { var err error - var c configuration + var c Configuration err = json.Unmarshal(config, &c) if err != nil { return nil, errors.Annotatef(err, "Invalid configuration file") } + // keep at least 100 mappings block->addresses to allow rollback + if c.BlockAddressesToKeep < 100 { + c.BlockAddressesToKeep = 100 + } transport := &http.Transport{ Dial: (&net.Dialer{KeepAlive: 600 * time.Second}).Dial, MaxIdleConns: 100, @@ -59,14 +63,13 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT } s := &BitcoinRPC{ - client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport}, - rpcURL: c.RPCURL, - user: c.RPCUser, - password: c.RPCPass, - ParseBlocks: c.Parse, - Subversion: c.Subversion, - zeroMQBinding: c.ZeroMQBinding, - pushHandler: pushHandler, + client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport}, + rpcURL: c.RPCURL, + user: c.RPCUser, + password: c.RPCPass, + ParseBlocks: c.Parse, + ChainConfig: &c, + pushHandler: pushHandler, } return s, nil @@ -82,7 +85,7 @@ func (b *BitcoinRPC) GetChainInfoAndInitializeMempool(bc bchain.BlockChain) (str return "", err } - mq, err := bchain.NewMQ(b.zeroMQBinding, b.pushHandler) + mq, err := bchain.NewMQ(b.ChainConfig.ZeroMQBinding, b.pushHandler) if err != nil { glog.Error("mq: ", err) return "", err @@ -105,7 +108,7 @@ func (b *BitcoinRPC) Initialize() error { params := GetChainParams(chainName) // always create parser - b.Parser = NewBitcoinParser(params) + b.Parser = NewBitcoinParser(params, b.ChainConfig) // parameters for getInfo request if params.Net == wire.MainNet { @@ -140,7 +143,7 @@ func (b *BitcoinRPC) GetNetworkName() string { } func (b *BitcoinRPC) GetSubversion() string { - return b.Subversion + return b.ChainConfig.Subversion } // getblockhash diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index 039542ad..84ee8d1b 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -2,6 +2,7 @@ package zec import ( "blockbook/bchain" + "blockbook/bchain/coins/btc" ) // ZCashParser handle @@ -10,8 +11,13 @@ type ZCashParser struct { } // NewZCAshParser returns new ZCAshParser instance -func NewZCashParser() *ZCashParser { - return &ZCashParser{&bchain.BaseParser{AddressFactory: bchain.NewBaseAddress}} +func NewZCashParser(c *btc.Configuration) *ZCashParser { + return &ZCashParser{ + &bchain.BaseParser{ + AddressFactory: bchain.NewBaseAddress, + BlockAddressesToKeep: c.BlockAddressesToKeep, + }, + } } // GetAddrIDFromVout returns internal address representation of given transaction output diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index ee63cb52..7b8db889 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -2,6 +2,7 @@ package zec import ( "blockbook/bchain" + "blockbook/bchain/coins/btc" "encoding/hex" "reflect" "testing" @@ -123,7 +124,7 @@ func TestPackTx(t *testing.T) { tx: testTx1, height: 292272, blockTime: 1521645728, - parser: NewZCashParser(), + parser: NewZCashParser(&btc.Configuration{}), }, want: testTxPacked1, wantErr: false, @@ -134,7 +135,7 @@ func TestPackTx(t *testing.T) { tx: testTx2, height: 292217, blockTime: 1521637604, - parser: NewZCashParser(), + parser: NewZCashParser(&btc.Configuration{}), }, want: testTxPacked2, wantErr: false, @@ -171,7 +172,7 @@ func TestUnpackTx(t *testing.T) { name: "zec-1", args: args{ packedTx: testTxPacked1, - parser: NewZCashParser(), + parser: NewZCashParser(&btc.Configuration{}), }, want: &testTx1, want1: 292272, @@ -181,7 +182,7 @@ func TestUnpackTx(t *testing.T) { name: "zec-2", args: args{ packedTx: testTxPacked2, - parser: NewZCashParser(), + parser: NewZCashParser(&btc.Configuration{}), }, want: &testTx2, want1: 292217, diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 31d943f5..c20aaf4c 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -31,7 +31,7 @@ func (z *ZCashRPC) Initialize() error { return err } - z.Parser = NewZCashParser() + z.Parser = NewZCashParser(z.ChainConfig) z.Testnet = false z.Network = "livenet" diff --git a/configs/bch-testnet.json b/configs/bch-testnet.json index 3b058d66..f518fac2 100644 --- a/configs/bch-testnet.json +++ b/configs/bch-testnet.json @@ -5,5 +5,6 @@ "rpcTimeout": 25, "parse": true, "zeroMQBinding": "tcp://127.0.0.1:48331", - "subversion": "/Bitcoin ABC:0.17.1/" + "subversion": "/Bitcoin ABC:0.17.1/", + "blockAddressesToKeep": 300 } diff --git a/configs/bch.json b/configs/bch.json index 1ae8d04d..55993b90 100644 --- a/configs/bch.json +++ b/configs/bch.json @@ -5,5 +5,6 @@ "rpcTimeout": 25, "parse": true, "zeroMQBinding": "tcp://127.0.0.1:38331", - "subversion": "/Bitcoin ABC:0.17.1/" + "subversion": "/Bitcoin ABC:0.17.1/", + "blockAddressesToKeep": 300 } diff --git a/configs/btc-testnet.json b/configs/btc-testnet.json index 46f5963a..5b8beab3 100644 --- a/configs/btc-testnet.json +++ b/configs/btc-testnet.json @@ -4,5 +4,6 @@ "rpcPass": "rpc", "rpcTimeout": 25, "parse": true, - "zeroMQBinding": "tcp://127.0.0.1:48330" + "zeroMQBinding": "tcp://127.0.0.1:48330", + "blockAddressesToKeep": 300 } diff --git a/configs/btc.json b/configs/btc.json index 59ad6826..538dd217 100644 --- a/configs/btc.json +++ b/configs/btc.json @@ -4,5 +4,6 @@ "rpcPass": "rpc", "rpcTimeout": 25, "parse": true, - "zeroMQBinding": "tcp://127.0.0.1:38330" + "zeroMQBinding": "tcp://127.0.0.1:38330", + "blockAddressesToKeep": 300 } diff --git a/configs/zec.json b/configs/zec.json index a68e27f5..431e8f2a 100644 --- a/configs/zec.json +++ b/configs/zec.json @@ -4,5 +4,6 @@ "rpcPass": "rpc", "rpcTimeout": 25, "parse": true, - "zeroMQBinding": "tcp://127.0.0.1:38332" + "zeroMQBinding": "tcp://127.0.0.1:38332", + "blockAddressesToKeep": 300 } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index a0960d35..0c28cb80 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -436,11 +436,6 @@ type testBitcoinParser struct { *btc.BitcoinParser } -// override btc.KeepBlockAddresses to keep only one blockaddress -func (p *testBitcoinParser) KeepBlockAddresses() int { - return 1 -} - // override PackTx and UnpackTx to default BaseParser functionality // BitcoinParser uses tx hex which is not available for the test transactions func (p *testBitcoinParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { @@ -481,7 +476,12 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { - d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}}) + d := setupRocksDB(t, &testBitcoinParser{ + BitcoinParser: &btc.BitcoinParser{ + BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, + Params: btc.GetChainParams("test"), + }, + }) defer closeAndDestroyRocksDB(t, d) // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column