diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 3c3e24c9..921263c3 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -43,6 +43,7 @@ func NewBlockChain(coin string, configfile string, pushHandler func(*bchain.MQMe if err != nil { return nil, err } + bc.Initialize(bchain.NewMempool(bc, metrics)) return &blockChainWithMetrics{b: bc, m: metrics}, nil } @@ -55,6 +56,10 @@ func (c *blockChainWithMetrics) observeRPCLatency(method string, start time.Time c.m.RPCLatency.With(common.Labels{"method": method, "error": err.Error()}).Observe(float64(time.Since(start)) / 1e6) // in milliseconds } +func (c *blockChainWithMetrics) Initialize(mempool *bchain.Mempool) error { + return c.b.Initialize(mempool) +} + func (c *blockChainWithMetrics) Shutdown() error { return c.b.Shutdown() } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 415082a2..692d1170 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -41,8 +41,8 @@ func (p *BitcoinBlockParser) GetUIDFromAddress(address string) ([]byte, error) { return p.AddressToOutputScript(address) } -func (p *BitcoinBlockParser) PackUID(script string) ([]byte, error) { - return hex.DecodeString(script) +func (p *BitcoinBlockParser) PackUID(str string) ([]byte, error) { + return hex.DecodeString(str) } func (p *BitcoinBlockParser) UnpackUID(buf []byte) string { diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index cc4e6c9b..ba4f62d5 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -24,7 +24,7 @@ type BitcoinRPC struct { rpcURL string user string password string - Parser *BitcoinBlockParser + Parser bchain.BlockChainParser Testnet bool Network string Mempool *bchain.Mempool @@ -64,26 +64,6 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(*bchain.MQMessage), ParseBlocks: c.Parse, metrics: metrics, } - chainName, err := s.GetBlockChainInfo() - if err != nil { - return nil, err - } - - // always create parser - s.Parser = &BitcoinBlockParser{ - Params: GetChainParams(chainName), - } - - // parameters for getInfo request - if s.Parser.Params.Net == wire.MainNet { - s.Testnet = false - s.Network = "livenet" - } else { - s.Testnet = true - s.Network = "testnet" - } - - glog.Info("rpc: block chain ", s.Parser.Params.Name) mq, err := bchain.NewMQ(c.ZeroMQBinding, pushHandler) if err != nil { @@ -92,11 +72,38 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(*bchain.MQMessage), } s.mq = mq - s.Mempool = bchain.NewMempool(s, metrics) - return s, nil } +func (b *BitcoinRPC) Initialize(mempool *bchain.Mempool) error { + b.Mempool = mempool + + chainName, err := b.GetBlockChainInfo() + if err != nil { + return err + } + + params := GetChainParams(chainName) + + // always create parser + b.Parser = &BitcoinBlockParser{ + Params: params, + } + + // parameters for getInfo request + if params.Net == wire.MainNet { + b.Testnet = false + b.Network = "livenet" + } else { + b.Testnet = true + b.Network = "testnet" + } + + glog.Info("rpc: block chain ", params.Name) + + return nil +} + func (b *BitcoinRPC) Shutdown() error { if b.mq != nil { if err := b.mq.Shutdown(); err != nil { @@ -188,12 +195,7 @@ type cmdGetBlockHeader struct { } `json:"params"` } -type resGetBlockHeaderRaw struct { - Error *bchain.RPCError `json:"error"` - Result string `json:"result"` -} - -type resGetBlockHeaderVerbose struct { +type resGetBlockHeader struct { Error *bchain.RPCError `json:"error"` Result bchain.BlockHeader `json:"result"` } @@ -233,12 +235,7 @@ type cmdGetRawTransaction struct { } `json:"params"` } -type resGetRawTransactionRaw struct { - Error *bchain.RPCError `json:"error"` - Result string `json:"result"` -} - -type resGetRawTransactionVerbose struct { +type resGetRawTransaction struct { Error *bchain.RPCError `json:"error"` Result bchain.Tx `json:"result"` } @@ -292,7 +289,7 @@ func (b *BitcoinRPC) GetBestBlockHash() (string, error) { res := resGetBestBlockHash{} req := cmdGetBestBlockHash{Method: "getbestblockhash"} - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return "", err @@ -309,7 +306,7 @@ func (b *BitcoinRPC) GetBestBlockHeight() (uint32, error) { res := resGetBlockCount{} req := cmdGetBlockCount{Method: "getblockcount"} - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return 0, err @@ -326,7 +323,7 @@ func (b *BitcoinRPC) GetBlockChainInfo() (string, error) { res := resGetBlockChainInfo{} req := cmdGetBlockChainInfo{Method: "getblockchaininfo"} - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return "", err @@ -344,7 +341,7 @@ func (b *BitcoinRPC) GetBlockHash(height uint32) (string, error) { res := resGetBlockHash{} req := cmdGetBlockHash{Method: "getblockhash"} req.Params.Height = height - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return "", errors.Annotatef(err, "height %v", height) @@ -359,11 +356,11 @@ func (b *BitcoinRPC) GetBlockHash(height uint32) (string, error) { func (b *BitcoinRPC) GetBlockHeader(hash string) (*bchain.BlockHeader, error) { glog.V(1).Info("rpc: getblockheader") - res := resGetBlockHeaderVerbose{} + res := resGetBlockHeader{} req := cmdGetBlockHeader{Method: "getblockheader"} req.Params.BlockHash = hash req.Params.Verbose = true - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, errors.Annotatef(err, "hash %v", hash) @@ -402,9 +399,6 @@ func (b *BitcoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) // getBlockWithoutHeader is an optimization - it does not call GetBlockHeader to get prev, next hashes // instead it sets to header only block hash and height passed in parameters func (b *BitcoinRPC) getBlockWithoutHeader(hash string, height uint32) (*bchain.Block, error) { - if !b.ParseBlocks { - return b.GetBlockFull(hash) - } data, err := b.GetBlockRaw(hash) if err != nil { return nil, err @@ -426,7 +420,7 @@ func (b *BitcoinRPC) GetBlockRaw(hash string) ([]byte, error) { req := cmdGetBlock{Method: "getblock"} req.Params.BlockHash = hash req.Params.Verbosity = 0 - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, errors.Annotatef(err, "hash %v", hash) @@ -446,7 +440,7 @@ func (b *BitcoinRPC) GetBlockList(hash string) (*bchain.Block, error) { req := cmdGetBlock{Method: "getblock"} req.Params.BlockHash = hash req.Params.Verbosity = 1 - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, errors.Annotatef(err, "hash %v", hash) @@ -478,7 +472,7 @@ func (b *BitcoinRPC) GetBlockFull(hash string) (*bchain.Block, error) { req := cmdGetBlock{Method: "getblock"} req.Params.BlockHash = hash req.Params.Verbosity = 2 - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, errors.Annotatef(err, "hash %v", hash) @@ -495,7 +489,7 @@ func (b *BitcoinRPC) GetMempool() ([]string, error) { res := resGetMempool{} req := cmdGetMempool{Method: "getrawmempool"} - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, err @@ -510,11 +504,11 @@ func (b *BitcoinRPC) GetMempool() ([]string, error) { func (b *BitcoinRPC) GetTransaction(txid string) (*bchain.Tx, error) { glog.V(1).Info("rpc: getrawtransaction ", txid) - res := resGetRawTransactionVerbose{} + res := resGetRawTransaction{} req := cmdGetRawTransaction{Method: "getrawtransaction"} req.Params.Txid = txid req.Params.Verbose = true - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) @@ -553,7 +547,7 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, e } else { req.Params.EstimateMode = "ECONOMICAL" } - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return 0, err @@ -571,7 +565,7 @@ func (b *BitcoinRPC) SendRawTransaction(tx string) (string, error) { res := resSendRawTransaction{} req := cmdSendRawTransaction{Method: "sendrawtransaction"} req.Params = []string{tx} - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return "", err @@ -590,7 +584,7 @@ func (b *BitcoinRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) Method: "getmempoolentry", Params: []string{txid}, } - err := b.observeRPCLatency(req.Method, func() error { return b.call(&req, &res) }) + err := b.Call(req.Method, &req, &res) if err != nil { return nil, err @@ -601,9 +595,9 @@ func (b *BitcoinRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) return res.Result, nil } -func (b *BitcoinRPC) observeRPCLatency(method string, fn func() error) error { +func (b *BitcoinRPC) Call(method string, req interface{}, res interface{}) error { start := time.Now() - err := fn() + err := b.call(req, res) if err == nil { b.metrics.RPCLatency.With(common.Labels{"method": method}).Observe(float64(time.Since(start)) / 1e6) // in milliseconds } diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go new file mode 100644 index 00000000..fab2b5ba --- /dev/null +++ b/bchain/coins/zec/zcashparser.go @@ -0,0 +1,46 @@ +package zec + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + + "github.com/btcsuite/btcd/chaincfg" +) + +// bitcoinwire parsing + +type ZCashBlockParser struct { + btc.BitcoinBlockParser +} + +// getChainParams contains network parameters for the main Bitcoin network, +// the regression test Bitcoin network, the test Bitcoin network and +// the simulation test Bitcoin network, in this order +func GetChainParams(chain string) *chaincfg.Params { + switch chain { + case "test": + return &chaincfg.TestNet3Params + case "regtest": + return &chaincfg.RegressionNetParams + } + return &chaincfg.MainNetParams +} + +func (p *ZCashBlockParser) GetUIDFromVout(output *bchain.Vout) string { + if len(output.ScriptPubKey.Addresses) != 1 { + return "" + } + return output.ScriptPubKey.Addresses[0] +} + +func (p *ZCashBlockParser) GetUIDFromAddress(address string) ([]byte, error) { + return p.PackUID(address) +} + +func (p *ZCashBlockParser) PackUID(str string) ([]byte, error) { + return []byte(str), nil +} + +func (p *ZCashBlockParser) UnpackUID(buf []byte) string { + return string(buf) +} diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 4f7e38db..5f6dd539 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -5,6 +5,11 @@ import ( "blockbook/bchain/coins/btc" "blockbook/common" "encoding/json" + + "github.com/btcsuite/btcd/wire" + + "github.com/golang/glog" + "github.com/juju/errors" ) type ZCashRPC struct { @@ -21,3 +26,155 @@ func NewZCashRPC(config json.RawMessage, pushHandler func(*bchain.MQMessage), me } return z, nil } + +func (z *ZCashRPC) Initialize(mempool *bchain.Mempool) error { + z.Mempool = mempool + + chainName, err := z.GetBlockChainInfo() + if err != nil { + return err + } + + params := GetChainParams(chainName) + + // always create parser + z.Parser = &ZCashBlockParser{ + btc.BitcoinBlockParser{ + Params: params, + }, + } + + // parameters for getInfo request + if params.Net == wire.MainNet { + z.Testnet = false + z.Network = "livenet" + } else { + z.Testnet = true + z.Network = "testnet" + } + + glog.Info("rpc: block chain ", params.Name) + + return nil +} + +type untypedArrayParams struct { + Method string `json:"method"` + Params []interface{} `json:"params"` +} + +// getblockhash + +type resGetBlockHash struct { + Error *bchain.RPCError `json:"error"` + Result string `json:"result"` +} + +// getblock + +type resGetBlockThin struct { + Error *bchain.RPCError `json:"error"` + Result bchain.ThinBlock `json:"result"` +} + +// getrawtransaction + +type resGetRawTransaction struct { + Error *bchain.RPCError `json:"error"` + Result bchain.Tx `json:"result"` +} + +// getblockheader + +type resGetBlockHeader struct { + Error *bchain.RPCError `json:"error"` + Result bchain.BlockHeader `json:"result"` +} + +// GetBlock returns block with given hash. +func (z *ZCashRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { + glog.V(1).Info("rpc: getblock (verbosity=1) ", hash) + + res := resGetBlockThin{} + req := untypedArrayParams{Method: "getblock"} + req.Params = append(req.Params, hash) + req.Params = append(req.Params, true) + err := z.Call(req.Method, &req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + + txs := make([]bchain.Tx, len(res.Result.Txids)) + for i, txid := range res.Result.Txids { + tx, err := z.GetTransaction(txid) + if err != nil { + return nil, err + } + txs[i] = *tx + } + block := &bchain.Block{ + BlockHeader: res.Result.BlockHeader, + Txs: txs, + } + return block, nil +} + +// GetTransaction returns a transaction by the transaction ID. +func (z *ZCashRPC) GetTransaction(txid string) (*bchain.Tx, error) { + glog.V(1).Info("rpc: getrawtransaction ", txid) + + res := resGetRawTransaction{} + req := untypedArrayParams{Method: "getrawtransaction"} + req.Params = append(req.Params, txid) + req.Params = append(req.Params, 1) + err := z.Call(req.Method, &req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "txid %v", txid) + } + if res.Error != nil { + return nil, errors.Annotatef(res.Error, "txid %v", txid) + } + return &res.Result, nil +} + +// GetBlockHash returns hash of block in best-block-chain at given height. +func (z *ZCashRPC) GetBlockHash(height uint32) (string, error) { + glog.V(1).Info("rpc: getblockhash ", height) + + res := resGetBlockHash{} + req := untypedArrayParams{Method: "getblockhash"} + req.Params = append(req.Params, height) + err := z.Call(req.Method, &req, &res) + + if err != nil { + return "", errors.Annotatef(err, "height %v", height) + } + if res.Error != nil { + return "", errors.Annotatef(res.Error, "height %v", height) + } + return res.Result, nil +} + +// GetBlockHeader returns header of block with given hash. +func (z *ZCashRPC) GetBlockHeader(hash string) (*bchain.BlockHeader, error) { + glog.V(1).Info("rpc: getblockheader") + + res := resGetBlockHeader{} + req := untypedArrayParams{Method: "getblockheader"} + req.Params = append(req.Params, hash) + req.Params = append(req.Params, true) + err := z.Call(req.Method, &req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + return &res.Result, nil +} diff --git a/bchain/coins/zec/zec.md b/bchain/coins/zec/zec.md new file mode 100644 index 00000000..f3adeecb --- /dev/null +++ b/bchain/coins/zec/zec.md @@ -0,0 +1,45 @@ +## Zcash Setup +Get Zcash client +``` +wget https://z.cash/downloads/zcash-1.0.15-linux64.tar.gz +tar xzf zcash-1.0.15-linux64.tar.gz +``` + +Run command to download the parameters used to create and verify shielded transactions: +``` +zcash-1.0.15/bin/zcash-fetch-params +``` + +Data are stored in */data/zec* , in folders */data/zec/zcash* for Zcash client data, */data/zec/blockbook* for Blockbook data. + +Create configuration file */data/zec/zcash/zcash.conf* with content +``` +daemon=1 +server=1 +rpcuser=rpc +rpcpassword=rpc +rpcport=8232 +txindex=1 +mainnet=1 +addnode=mainnet.z.cash +``` + +Create script *run-zec-zcashd.sh* that starts the zcashd daemon with increased rpcworkqueue and configured zeromq +``` +#!/bin/bash + +zcash-1.0.15/bin/zcashd -datadir=/data/zec/zcash -rpcworkqueue=32 -zmqpubhashblock=tcp://127.0.0.1:8234 -zmqpubrawblock=tcp://127.0.0.1:8234 -zmqpubhashtx=tcp://127.0.0.1:8234 -zmqpubrawtx=tcp://127.0.0.1:8234 +``` + +Run the *run-zec-zcashd.sh* to get initial import of data. + +Create *run-zec-blockbook.sh* script that starts blockbook +``` +#!/bin/bash +./blockbook -path=/data/zec/blockbook/db -sync -parse -rpcurl=http://127.0.0.1:8232 -httpserver=:8235 -socketio=:8236 -certfile=server/testcert -zeromq=tcp://127.0.0.1:8234 -explorer=https://zec-bitcore1.trezor.io -coin=zec $1 +``` + +To run blockbook with logging to file (run with nohup or daemonize using screen) +``` +./run-zec-blockbook.sh 2> /data/zec/blockbook/blockbook.log +``` diff --git a/bchain/mempool.go b/bchain/mempool.go index 37155fda..a37b6c93 100644 --- a/bchain/mempool.go +++ b/bchain/mempool.go @@ -27,7 +27,6 @@ type inputOutput struct { // Mempool is mempool handle. type Mempool struct { chain BlockChain - chainParser BlockChainParser mux sync.Mutex txToInputOutput map[string]inputOutput scriptToTx map[string][]outpoint // TODO rename all occurences @@ -37,18 +36,19 @@ type Mempool struct { // NewMempool creates new mempool handler. func NewMempool(chain BlockChain, metrics *common.Metrics) *Mempool { - return &Mempool{chain: chain, chainParser: chain.GetChainParser(), metrics: metrics} + return &Mempool{chain: chain, metrics: metrics} } // GetTransactions returns slice of mempool transactions for given output script. func (m *Mempool) GetTransactions(address string) ([]string, error) { m.mux.Lock() defer m.mux.Unlock() - buf, err := m.chainParser.GetUIDFromAddress(address) + parser := m.chain.GetChainParser() + buf, err := parser.GetUIDFromAddress(address) if err != nil { return nil, err } - outid := m.chainParser.UnpackUID(buf) + outid := parser.UnpackUID(buf) outpoints := m.scriptToTx[outid] txs := make([]string, 0, len(outpoints)+len(outpoints)/2) for _, o := range outpoints { @@ -86,6 +86,7 @@ func (m *Mempool) Resync(onNewTxAddr func(txid string, addr string)) error { m.metrics.MempoolResyncErrors.With(common.Labels{"error": err.Error()}).Inc() return err } + parser := m.chain.GetChainParser() newTxToInputOutput := make(map[string]inputOutput, len(m.txToInputOutput)+1) newScriptToTx := make(map[string][]outpoint, len(m.scriptToTx)+1) newInputs := make(map[outpoint]string, len(m.inputs)+1) @@ -100,7 +101,7 @@ func (m *Mempool) Resync(onNewTxAddr func(txid string, addr string)) error { } io.outputs = make([]scriptIndex, 0, len(tx.Vout)) for _, output := range tx.Vout { - outid := m.chainParser.GetUIDFromVout(&output) + outid := parser.GetUIDFromVout(&output) if outid != "" { io.outputs = append(io.outputs, scriptIndex{outid, output.N}) } diff --git a/bchain/types.go b/bchain/types.go index 8d982021..6e665364 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -86,7 +86,8 @@ func (e *RPCError) Error() string { } type BlockChain interface { - // cleanup + // life-cycle methods + Initialize(mempool *Mempool) error Shutdown() error // chain info IsTestnet() bool diff --git a/db/rocksdb.go b/db/rocksdb.go index 43d01260..50ce4438 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -266,7 +266,7 @@ func (d *RocksDB) writeOutputs(wb *gorocksdb.WriteBatch, block *bchain.Block, op for outid, outpoints := range records { bOutid, err := d.chainParser.PackUID(outid) if err != nil { - glog.Warningf("rocksdb: packOutputID: %v - %d %s", err, block.Height, outid) + glog.Warningf("rocksdb: packUID: %v - %d %s", err, block.Height, outid) continue } key, err := packOutputKey(bOutid, block.Height)