From b464f282a9203905a34dc7928948470e1f5c0fa8 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 24 Jul 2018 15:58:37 +0200 Subject: [PATCH 001/123] Use big.Int for all amounts --- api/types.go | 59 +++++++++++------------ api/worker.go | 63 ++++++++++++------------ bchain/baseparser.go | 60 +++++++++++++++++++++-- bchain/coins/bch/bcashrpc.go | 14 ++++-- bchain/coins/blockchain.go | 5 +- bchain/coins/btc/bitcoinparser.go | 6 ++- bchain/coins/btc/bitcoinrpc.go | 72 +++++++++++++++++++--------- bchain/coins/dash/dashrpc.go | 6 +-- bchain/coins/dogecoin/dogecoinrpc.go | 6 +-- bchain/coins/eth/ethparser.go | 20 ++++---- bchain/coins/eth/ethrpc.go | 10 ++-- bchain/coins/litecoin/litecoinrpc.go | 6 +-- bchain/coins/namecoin/namecoinrpc.go | 6 +-- bchain/coins/vertcoin/vertcoinrpc.go | 6 +-- bchain/coins/zec/zcashrpc.go | 9 +--- bchain/tx.pb.go | 52 ++++++++++---------- bchain/tx.proto | 2 +- bchain/types.go | 21 +++++--- server/public.go | 6 ++- server/socketio.go | 19 ++++---- 20 files changed, 261 insertions(+), 187 deletions(-) diff --git a/api/types.go b/api/types.go index b605bd30..287e04af 100644 --- a/api/types.go +++ b/api/types.go @@ -1,5 +1,7 @@ package api +import "math/big" + type ScriptSig struct { Hex string `json:"hex"` Asm string `json:"asm,omitempty"` @@ -12,8 +14,8 @@ type Vin struct { N int `json:"n"` ScriptSig ScriptSig `json:"scriptSig"` Addr string `json:"addr"` - ValueSat int64 `json:"valueSat"` - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` } type ScriptPubKey struct { @@ -23,7 +25,8 @@ type ScriptPubKey struct { Type string `json:"type,omitempty"` } type Vout struct { - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` N int `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` SpentTxID string `json:"spentTxId,omitempty"` @@ -32,34 +35,30 @@ type Vout struct { } type Tx struct { - Txid string `json:"txid"` - Version int32 `json:"version,omitempty"` - Locktime uint32 `json:"locktime,omitempty"` - Vin []Vin `json:"vin"` - Vout []Vout `json:"vout"` - Blockhash string `json:"blockhash,omitempty"` - Blockheight int `json:"blockheight"` - Confirmations uint32 `json:"confirmations"` - Time int64 `json:"time,omitempty"` - Blocktime int64 `json:"blocktime"` - ValueOut float64 `json:"valueOut"` - Size int `json:"size,omitempty"` - ValueIn float64 `json:"valueIn"` - Fees float64 `json:"fees"` - WithSpends bool `json:"withSpends,omitempty"` + Txid string `json:"txid"` + Version int32 `json:"version,omitempty"` + Locktime uint32 `json:"locktime,omitempty"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` + Blockhash string `json:"blockhash,omitempty"` + Blockheight int `json:"blockheight"` + Confirmations uint32 `json:"confirmations"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime"` + ValueOut string `json:"valueOut"` + Size int `json:"size,omitempty"` + ValueIn string `json:"valueIn"` + Fees string `json:"fees"` + WithSpends bool `json:"withSpends,omitempty"` } type Address struct { - AddrStr string `json:"addrStr"` - Balance float64 `json:"balance"` - BalanceSat int64 `json:"balanceSat"` - TotalReceived float64 `json:"totalReceived"` - TotalReceivedSat int64 `json:"totalReceivedSat"` - TotalSent float64 `json:"totalSent"` - TotalSentSat int64 `json:"totalSentSat"` - UnconfirmedBalance float64 `json:"unconfirmedBalance"` - UnconfirmedBalanceSat int64 `json:"unconfirmedBalanceSat"` - UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` - TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"transactions"` + AddrStr string `json:"addrStr"` + Balance string `json:"balance"` + TotalReceived string `json:"totalReceived"` + TotalSent string `json:"totalSent"` + UnconfirmedBalance string `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*Tx `json:"transactions"` } diff --git a/api/worker.go b/api/worker.go index 4ad421d3..ea2870e6 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "math/big" "github.com/golang/glog" ) @@ -44,7 +45,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) return nil, err } } - var valIn, valOut, fees float64 + var valInSat, valOutSat, feesSat big.Int vins := make([]Vin, len(bchainTx.Vin)) for i := range bchainTx.Vin { bchainVin := &bchainTx.Vin[i] @@ -61,9 +62,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) } if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] - vin.Value = vout.Value - valIn += vout.Value - vin.ValueSat = int64(vout.Value*1E8 + 0.5) + vin.ValueSat = vout.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) + valInSat.Add(&valInSat, &vout.ValueSat) if vout.Address != nil { a := vout.Address.String() vin.Addr = a @@ -76,8 +77,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) bchainVout := &bchainTx.Vout[i] vout := &vouts[i] vout.N = i - vout.Value = bchainVout.Value - valOut += bchainVout.Value + vout.ValueSat = bchainVout.ValueSat + vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) + valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses if spendingTx { @@ -85,9 +87,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) } } // for coinbase transactions valIn is 0 - fees = valIn - valOut - if fees < 0 { - fees = 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) } // for now do not return size, we would have to compute vsize of segwit transactions // size:=len(bchainTx.Hex) / 2 @@ -96,13 +98,13 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) Blockheight: int(height), Blocktime: bchainTx.Blocktime, Confirmations: bchainTx.Confirmations, - Fees: fees, + Fees: w.chainParser.AmountToDecimalString(&feesSat), Locktime: bchainTx.LockTime, WithSpends: spendingTx, Time: bchainTx.Time, Txid: bchainTx.Txid, - ValueIn: valIn, - ValueOut: valOut, + ValueIn: w.chainParser.AmountToDecimalString(&valInSat), + ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), Version: bchainTx.Version, Vin: vins, Vout: vouts, @@ -131,26 +133,26 @@ func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return txids, nil } -func (t *Tx) getAddrVoutValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVoutValue(addrID string) *big.Int { + var val big.Int for _, vout := range t.Vout { for _, a := range vout.ScriptPubKey.Addresses { if a == addrID { - val += vout.Value + val.Add(&val, &vout.ValueSat) } } } - return val + return &val } -func (t *Tx) getAddrVinValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVinValue(addrID string) *big.Int { + var val big.Int for _, vin := range t.Vin { if vin.Addr == addrID { - val += vin.Value + val.Add(&val, &vin.ValueSat) } } - return val + return &val } // UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions @@ -191,14 +193,14 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { } txs := make([]*Tx, len(txm)+lc) txi := 0 - var uBal, bal, totRecv, totSent float64 + var uBalSat, balSat, totRecvSat, totSentSat big.Int for _, tx := range txm { tx, err := w.GetTransaction(tx, bestheight, false) // mempool transaction may fail if err != nil { glog.Error("GetTransaction ", tx, ": ", err) } else { - uBal = tx.getAddrVoutValue(addrID) - tx.getAddrVinValue(addrID) + uBalSat.Sub(tx.getAddrVoutValue(addrID), tx.getAddrVinValue(addrID)) txs[txi] = tx txi++ } @@ -216,26 +218,23 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { if err != nil { return nil, err } else { - totRecv += tx.getAddrVoutValue(addrID) - totSent += tx.getAddrVinValue(addrID) + totRecvSat.Add(&totRecvSat, tx.getAddrVoutValue(addrID)) + totSentSat.Add(&totSentSat, tx.getAddrVinValue(addrID)) if i >= from && i < to { txs[txi] = tx txi++ } } } - bal = totRecv - totSent + balSat.Sub(&totRecvSat, &totSentSat) r := &Address{ AddrStr: addrID, - Balance: bal, - BalanceSat: int64(bal*1E8 + 0.5), - TotalReceived: totRecv, - TotalReceivedSat: int64(totRecv*1E8 + 0.5), - TotalSent: totSent, - TotalSentSat: int64(totSent*1E8 + 0.5), + Balance: w.chainParser.AmountToDecimalString(&balSat), + TotalReceived: w.chainParser.AmountToDecimalString(&totRecvSat), + TotalSent: w.chainParser.AmountToDecimalString(&totSentSat), Transactions: txs[:txi], TxApperances: len(txc), - UnconfirmedBalance: uBal, + UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), UnconfirmedTxApperances: len(txm), } glog.Info(addrID, " finished") diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 8f7bc08a..d8d04fef 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -3,6 +3,8 @@ package bchain import ( "encoding/hex" "encoding/json" + "math/big" + "strings" "github.com/gogo/protobuf/proto" "github.com/juju/errors" @@ -14,6 +16,7 @@ type AddressFactoryFunc func(string) (Address, error) type BaseParser struct { AddressFactory AddressFactoryFunc BlockAddressesToKeep int + AmountDecimalPoint int } // AddressToOutputScript converts address to ScriptPubKey - currently not implemented @@ -36,7 +39,47 @@ func (p *BaseParser) ParseTx(b []byte) (*Tx, error) { return nil, errors.New("ParseTx: not implemented") } -// ParseTxFromJson parses JSON message containing transaction and returs Tx struct +const zeros = "0000000000000000000000000000000000000000" + +// AmountToBigInt converts amount in json.Number (string) to big.Int +// it uses string operations to avoid problems with rounding +func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { + var r big.Int + s := string(n) + i := strings.IndexByte(s, '.') + if i == -1 { + s = s + zeros[:p.AmountDecimalPoint] + } else { + z := p.AmountDecimalPoint - len(s) + i + 1 + if z > 0 { + s = s[:i] + s[i+1:] + zeros[:z] + } else { + s = s[:i] + s[i+1:len(s)+z] + } + } + if _, ok := r.SetString(s, 10); !ok { + return r, errors.New("AmountToBigInt: failed to convert") + } + return r, nil +} + +// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place +func (p *BaseParser) AmountToDecimalString(a *big.Int) string { + s := a.String() + if len(s) <= p.AmountDecimalPoint { + s = zeros[:p.AmountDecimalPoint-len(s)+1] + s + } + i := len(s) - p.AmountDecimalPoint + ad := strings.TrimRight(s[i:], "0") + if len(ad) > 0 { + s = s[:i] + "." + ad + } else { + s = s[:i] + } + return s +} + +// ParseTxFromJson parses JSON message containing transaction and returns Tx struct func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { var tx Tx err := json.Unmarshal(msg, &tx) @@ -44,13 +87,18 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { return nil, err } - for i, vout := range tx.Vout { + for i := range tx.Vout { + vout := &tx.Vout[i] + vout.ValueSat, err = p.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } if len(vout.ScriptPubKey.Addresses) == 1 { a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) if err != nil { return nil, err } - tx.Vout[i].Address = a + vout.Address = a } } @@ -127,7 +175,7 @@ func (p *BaseParser) PackTx(tx *Tx, height uint32, blockTime int64) ([]byte, err Addresses: vo.ScriptPubKey.Addresses, N: vo.N, ScriptPubKeyHex: hex, - Value: vo.Value, + ValueSat: vo.ValueSat.Bytes(), } } pt := &ProtoTransaction{ @@ -176,13 +224,15 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } vout := make([]Vout, len(pt.Vout)) for i, pto := range pt.Vout { + var vs big.Int + vs.SetBytes(pto.ValueSat) vout[i] = Vout{ N: pto.N, ScriptPubKey: ScriptPubKey{ Addresses: pto.Addresses, Hex: hex.EncodeToString(pto.ScriptPubKeyHex), }, - Value: pto.Value, + ValueSat: vs, } if len(pto.Addresses) == 1 { a, err := p.AddressFactory(pto.Addresses[0]) diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 68af837a..5dd9b141 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -5,6 +5,7 @@ import ( "blockbook/bchain/coins/btc" "encoding/hex" "encoding/json" + "math/big" "github.com/cpacia/bchutil" "github.com/golang/glog" @@ -132,7 +133,7 @@ func (b *BCashRPC) GetBlockFull(hash string) (*bchain.Block, error) { } // EstimateSmartFee returns fee estimation. -func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := btc.ResEstimateSmartFee{} @@ -141,13 +142,18 @@ func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, err // conservative param is omitted err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } func isErrBlockNotFound(err *bchain.RPCError) bool { diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index b1611161..d9e8eae4 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/big" "reflect" "time" @@ -172,12 +173,12 @@ func (c *blockChainWithMetrics) GetTransactionForMempool(txid string) (v *bchain return c.b.GetTransactionForMempool(txid) } -func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateSmartFee", s, err) }(time.Now()) return c.b.EstimateSmartFee(blocks, conservative) } -func (c *blockChainWithMetrics) EstimateFee(blocks int) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateFee(blocks int) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateFee", s, err) }(time.Now()) return c.b.EstimateFee(blocks) } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 861c7f31..767e8d08 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" vlq "github.com/bsm/go-vlq" "github.com/btcsuite/btcd/blockchain" @@ -30,6 +31,7 @@ func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser &bchain.BaseParser{ AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, params, outputScriptToAddresses, @@ -123,8 +125,10 @@ func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.T // missing: Asm, // missing: Type, } + var vs big.Int + vs.SetInt64(out.Value) vout[i] = bchain.Vout{ - Value: float64(out.Value) / 1E8, + ValueSat: vs, N: uint32(i), ScriptPubKey: s, } diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index ac482b8b..8f9fb067 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -8,6 +8,7 @@ import ( "encoding/json" "io" "io/ioutil" + "math/big" "net" "net/http" "time" @@ -36,18 +37,20 @@ type BitcoinRPC struct { } type Configuration struct { - CoinName string `json:"coin_name"` - RPCURL string `json:"rpcURL"` - RPCUser string `json:"rpcUser"` - RPCPass string `json:"rpcPass"` - RPCTimeout int `json:"rpcTimeout"` - Parse bool `json:"parse"` - ZeroMQBinding string `json:"zeroMQBinding"` - Subversion string `json:"subversion"` - BlockAddressesToKeep int `json:"blockAddressesToKeep"` - MempoolWorkers int `json:"mempoolWorkers"` - MempoolSubWorkers int `json:"mempoolSubWorkers"` - AddressFormat string `json:"addressFormat"` + CoinName string `json:"coin_name"` + RPCURL string `json:"rpcURL"` + RPCUser string `json:"rpcUser"` + RPCPass string `json:"rpcPass"` + RPCTimeout int `json:"rpcTimeout"` + Parse bool `json:"parse"` + ZeroMQBinding string `json:"zeroMQBinding"` + Subversion string `json:"subversion"` + BlockAddressesToKeep int `json:"blockAddressesToKeep"` + MempoolWorkers int `json:"mempoolWorkers"` + MempoolSubWorkers int `json:"mempoolSubWorkers"` + AddressFormat string `json:"addressFormat"` + SupportsEstimateFee bool `json:"supportsEstimateFee"` + SupportsEstimateSmartFee bool `json:"supportsEstimateSmartFee"` } // NewBitcoinRPC returns new BitcoinRPC instance. @@ -69,6 +72,9 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT if c.MempoolSubWorkers < 1 { c.MempoolSubWorkers = 1 } + // btc supports both calls, other coins overriding BitcoinRPC can change this + c.SupportsEstimateFee = true + c.SupportsEstimateSmartFee = true transport := &http.Transport{ Dial: (&net.Dialer{KeepAlive: 600 * time.Second}).Dial, @@ -301,8 +307,8 @@ type CmdEstimateSmartFee struct { type ResEstimateSmartFee struct { Error *bchain.RPCError `json:"error"` Result struct { - Feerate float64 `json:"feerate"` - Blocks int `json:"blocks"` + Feerate json.Number `json:"feerate"` + Blocks int `json:"blocks"` } `json:"result"` } @@ -317,7 +323,7 @@ type CmdEstimateFee struct { type ResEstimateFee struct { Error *bchain.RPCError `json:"error"` - Result float64 `json:"result"` + Result json.Number `json:"result"` } // sendrawtransaction @@ -616,7 +622,12 @@ func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]string, error) { } // EstimateSmartFee returns fee estimation. -func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + // use EstimateFee if EstimateSmartFee is not supported + if !b.ChainConfig.SupportsEstimateSmartFee && b.ChainConfig.SupportsEstimateFee { + return b.EstimateFee(blocks) + } + glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := ResEstimateSmartFee{} @@ -629,17 +640,27 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, e } err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } // EstimateFee returns fee estimation. -func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { +func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { + // use EstimateSmartFee if EstimateFee is not supported + if !b.ChainConfig.SupportsEstimateFee && b.ChainConfig.SupportsEstimateSmartFee { + return b.EstimateSmartFee(blocks, true) + } + glog.V(1).Info("rpc: estimatefee ", blocks) res := ResEstimateFee{} @@ -647,13 +668,18 @@ func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { req.Params.Blocks = blocks err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result, nil + r, err = b.Parser.AmountToBigInt(res.Result) + if err != nil { + return r, err + } + return r, nil } // SendRawTransaction sends raw transaction. diff --git a/bchain/coins/dash/dashrpc.go b/bchain/coins/dash/dashrpc.go index 4c884f08..dec0ee49 100644 --- a/bchain/coins/dash/dashrpc.go +++ b/bchain/coins/dash/dashrpc.go @@ -24,6 +24,7 @@ func NewDashRPC(config json.RawMessage, pushHandler func(bchain.NotificationType b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -53,8 +54,3 @@ func (b *DashRPC) Initialize() error { return nil } - -// EstimateSmartFee returns fee estimation. -func (b *DashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/dogecoin/dogecoinrpc.go b/bchain/coins/dogecoin/dogecoinrpc.go index d0ebb31c..1fc21618 100644 --- a/bchain/coins/dogecoin/dogecoinrpc.go +++ b/bchain/coins/dogecoin/dogecoinrpc.go @@ -24,6 +24,7 @@ func NewDogecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *DogecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *DogecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index fe361258..82d08153 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -22,7 +22,11 @@ type EthereumParser struct { // NewEthereumParser returns new EthereumParser instance func NewEthereumParser() *EthereumParser { - return &EthereumParser{&bchain.BaseParser{AddressFactory: bchain.NewBaseAddress}} + return &EthereumParser{&bchain.BaseParser{ + AddressFactory: bchain.NewBaseAddress, + BlockAddressesToKeep: 0, + AmountDecimalPoint: 18, + }} } type rpcTransaction struct { @@ -86,6 +90,10 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } tx.BlockHash = bh h := hex.EncodeToString(b) + vs, err := hexutil.DecodeBig(tx.Value) + if err != nil { + return nil, err + } return &bchain.Tx{ Blocktime: blocktime, Confirmations: confirmations, @@ -105,8 +113,8 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma }, Vout: []bchain.Vout{ { - N: 0, // there is always up to one To address - // Value - cannot be set, it does not fit precisely to float64 + N: 0, // there is always up to one To address + ValueSat: *vs, ScriptPubKey: bchain.ScriptPubKey{ // Hex Addresses: ta, @@ -286,9 +294,3 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) { func (p *EthereumParser) IsUTXOChain() bool { return false } - -// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column -// do not use the blockaddresses for eth -func (p *EthereumParser) KeepBlockAddresses() int { - return 0 -} diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index d985e74c..2dda7005 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -478,12 +478,12 @@ func (b *EthereumRPC) GetMempool() ([]string, error) { } // EstimateFee returns fee estimation. -func (b *EthereumRPC) EstimateFee(blocks int) (float64, error) { +func (b *EthereumRPC) EstimateFee(blocks int) (big.Int, error) { return b.EstimateSmartFee(blocks, true) } // EstimateSmartFee returns fee estimation. -func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() // TODO - what parameters of msg to use to get better estimate, maybe more data from the wallet are needed @@ -492,10 +492,12 @@ func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, To: &a, } g, err := b.client.EstimateGas(ctx, msg) + var r big.Int if err != nil { - return 0, err + return r, err } - return float64(g), nil + r.SetUint64(g) + return r, nil } // SendRawTransaction sends raw transaction. diff --git a/bchain/coins/litecoin/litecoinrpc.go b/bchain/coins/litecoin/litecoinrpc.go index 0d2ad15f..e7345251 100644 --- a/bchain/coins/litecoin/litecoinrpc.go +++ b/bchain/coins/litecoin/litecoinrpc.go @@ -24,6 +24,7 @@ func NewLitecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *LitecoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *LitecoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/namecoin/namecoinrpc.go b/bchain/coins/namecoin/namecoinrpc.go index faa14fd3..6ab0b0aa 100644 --- a/bchain/coins/namecoin/namecoinrpc.go +++ b/bchain/coins/namecoin/namecoinrpc.go @@ -24,6 +24,7 @@ func NewNamecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *NamecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *NamecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/vertcoin/vertcoinrpc.go b/bchain/coins/vertcoin/vertcoinrpc.go index 878b9682..f282dc78 100644 --- a/bchain/coins/vertcoin/vertcoinrpc.go +++ b/bchain/coins/vertcoin/vertcoinrpc.go @@ -24,6 +24,7 @@ func NewVertcoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *VertcoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *VertcoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 495fa7af..b2c45fe5 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -22,6 +22,7 @@ func NewZCashRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp BitcoinRPC: b.(*btc.BitcoinRPC), } z.RPCMarshaler = btc.JSONMarshalerV1{} + z.ChainConfig.SupportsEstimateSmartFee = false return z, nil } @@ -114,14 +115,6 @@ func (z *ZCashRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { return z.GetTransaction(txid) } -// EstimateSmartFee returns fee estimation. -func (z *ZCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - glog.V(1).Info("rpc: estimatesmartfee") - - // return z.estimateFee(blocks) - return z.EstimateFee(blocks) -} - // GetMempoolEntry returns mempool data for given transaction func (z *ZCashRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { return nil, errors.New("GetMempoolEntry: not implemented") diff --git a/bchain/tx.pb.go b/bchain/tx.pb.go index dc5a0952..2115993a 100644 --- a/bchain/tx.pb.go +++ b/bchain/tx.pb.go @@ -148,7 +148,7 @@ func (m *ProtoTransaction_VinType) GetAddresses() []string { } type ProtoTransaction_VoutType struct { - Value float64 `protobuf:"fixed64,1,opt,name=Value" json:"Value,omitempty"` + ValueSat []byte `protobuf:"bytes,1,opt,name=ValueSat,proto3" json:"ValueSat,omitempty"` N uint32 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` ScriptPubKeyHex []byte `protobuf:"bytes,3,opt,name=ScriptPubKeyHex,proto3" json:"ScriptPubKeyHex,omitempty"` Addresses []string `protobuf:"bytes,4,rep,name=Addresses" json:"Addresses,omitempty"` @@ -159,11 +159,11 @@ func (m *ProtoTransaction_VoutType) String() string { return proto.Co func (*ProtoTransaction_VoutType) ProtoMessage() {} func (*ProtoTransaction_VoutType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 1} } -func (m *ProtoTransaction_VoutType) GetValue() float64 { +func (m *ProtoTransaction_VoutType) GetValueSat() []byte { if m != nil { - return m.Value + return m.ValueSat } - return 0 + return nil } func (m *ProtoTransaction_VoutType) GetN() uint32 { @@ -196,26 +196,26 @@ func init() { func init() { proto.RegisterFile("tx.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 330 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x4e, 0xc2, 0x40, - 0x10, 0x86, 0xb3, 0xb4, 0x14, 0x18, 0x21, 0x92, 0x89, 0x31, 0x0d, 0xf1, 0x50, 0x39, 0xf5, 0xd4, - 0x03, 0xc6, 0x07, 0x50, 0x2f, 0x24, 0x1a, 0x42, 0x16, 0xd2, 0x7b, 0x5b, 0x36, 0xb0, 0x11, 0x77, - 0xb1, 0xdd, 0x1a, 0x78, 0x16, 0x1f, 0xc1, 0x97, 0x34, 0x3b, 0x2d, 0x45, 0x48, 0xbc, 0xed, 0xff, - 0xef, 0x4c, 0xff, 0x6f, 0xff, 0x14, 0xba, 0x66, 0x1f, 0xed, 0x72, 0x6d, 0x34, 0x7a, 0x69, 0xb6, - 0x49, 0xa4, 0x1a, 0x7f, 0xbb, 0x30, 0x9c, 0x5b, 0x67, 0x99, 0x27, 0xaa, 0x48, 0x32, 0x23, 0xb5, - 0x42, 0x04, 0x77, 0xb9, 0x97, 0x2b, 0x9f, 0x05, 0x2c, 0xec, 0x73, 0x3a, 0xe3, 0x10, 0x9c, 0xa9, - 0xd8, 0xfb, 0x2d, 0xb2, 0xec, 0x11, 0xef, 0xa0, 0xf7, 0xbc, 0xd5, 0xd9, 0xbb, 0x91, 0x1f, 0xc2, - 0x77, 0x02, 0x16, 0xba, 0xfc, 0x64, 0xe0, 0x08, 0xba, 0x6f, 0xc7, 0x4b, 0x37, 0x60, 0xe1, 0x80, - 0x37, 0x1a, 0x6f, 0xc1, 0x9b, 0x0a, 0xb9, 0xde, 0x18, 0xbf, 0x4d, 0x37, 0xb5, 0xc2, 0x09, 0x38, - 0xb1, 0x54, 0xbe, 0x17, 0x38, 0xe1, 0xd5, 0x24, 0x88, 0x2a, 0xc4, 0xe8, 0x12, 0x2f, 0x8a, 0xa5, - 0x5a, 0x1e, 0x76, 0x82, 0xdb, 0x61, 0x7c, 0x04, 0x37, 0xd6, 0xa5, 0xf1, 0x3b, 0xb4, 0x74, 0xff, - 0xff, 0x92, 0x2e, 0x0d, 0x6d, 0xd1, 0xf8, 0xe8, 0x87, 0x41, 0xa7, 0xfe, 0x8e, 0x45, 0x7d, 0xd1, - 0x52, 0xa5, 0x49, 0x21, 0xe8, 0xc9, 0x3d, 0xde, 0xe8, 0xa6, 0x8a, 0xd6, 0x9f, 0x2a, 0xb0, 0x8e, - 0x74, 0x08, 0x9e, 0xce, 0x38, 0x86, 0xfe, 0x22, 0xcb, 0xe5, 0xce, 0x2c, 0xe4, 0xda, 0xf6, 0xe4, - 0xd2, 0xfc, 0x99, 0x67, 0x73, 0x16, 0xe2, 0xb3, 0x14, 0x2a, 0x13, 0xf5, 0xc3, 0x1b, 0x6d, 0xcb, - 0x7c, 0x5a, 0xad, 0x72, 0x51, 0x14, 0xa2, 0xa0, 0x02, 0x7a, 0xfc, 0x64, 0x8c, 0xbe, 0xa0, 0x7b, - 0xe4, 0xc7, 0x1b, 0x68, 0xc7, 0xc9, 0xb6, 0xac, 0x50, 0x19, 0xaf, 0x04, 0xf6, 0x81, 0xcd, 0x08, - 0x72, 0xc0, 0xd9, 0x0c, 0x43, 0xb8, 0xae, 0x92, 0xe7, 0x65, 0xfa, 0x2a, 0x0e, 0x16, 0xc8, 0x21, - 0xa0, 0x4b, 0xfb, 0x3c, 0xd7, 0xbd, 0xc8, 0x4d, 0x3d, 0xfa, 0x59, 0x1e, 0x7e, 0x03, 0x00, 0x00, - 0xff, 0xff, 0x64, 0x1d, 0x38, 0xac, 0x38, 0x02, 0x00, 0x00, + // 331 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xc1, 0x6e, 0xea, 0x30, + 0x10, 0x94, 0x89, 0x5f, 0x80, 0x7d, 0xa0, 0x87, 0xf6, 0xf0, 0x14, 0xa1, 0x1e, 0x52, 0x4e, 0x39, + 0xe5, 0x40, 0xd5, 0x0f, 0x68, 0x7b, 0x41, 0x6a, 0x85, 0x90, 0x83, 0x72, 0x4f, 0x82, 0x05, 0x56, + 0xa9, 0x4d, 0x13, 0x47, 0x02, 0xa9, 0x3f, 0xd3, 0x73, 0x7f, 0xb2, 0xf2, 0x12, 0x42, 0x41, 0xea, + 0x6d, 0x67, 0xbc, 0xe3, 0x19, 0x4f, 0x02, 0x3d, 0xbb, 0x8f, 0x77, 0xa5, 0xb1, 0x06, 0xfd, 0xbc, + 0xd8, 0x64, 0x4a, 0x4f, 0x3e, 0x39, 0x8c, 0x16, 0x8e, 0x59, 0x96, 0x99, 0xae, 0xb2, 0xc2, 0x2a, + 0xa3, 0x11, 0x81, 0x2f, 0xf7, 0x6a, 0x15, 0xb0, 0x90, 0x45, 0x03, 0x41, 0x33, 0x8e, 0xc0, 0x9b, + 0xc9, 0x7d, 0xd0, 0x21, 0xca, 0x8d, 0x78, 0x03, 0xfd, 0xc7, 0xad, 0x29, 0x5e, 0xad, 0x7a, 0x93, + 0x81, 0x17, 0xb2, 0x88, 0x8b, 0x33, 0x81, 0x63, 0xe8, 0xbd, 0x9c, 0x0e, 0x79, 0xc8, 0xa2, 0xa1, + 0x68, 0x31, 0xfe, 0x07, 0x7f, 0x26, 0xd5, 0x7a, 0x63, 0x83, 0x3f, 0x74, 0xd2, 0x20, 0x9c, 0x82, + 0x97, 0x2a, 0x1d, 0xf8, 0xa1, 0x17, 0xfd, 0x9d, 0x86, 0xf1, 0x31, 0x62, 0x7c, 0x1d, 0x2f, 0x4e, + 0x95, 0x5e, 0x1e, 0x76, 0x52, 0xb8, 0x65, 0xbc, 0x07, 0x9e, 0x9a, 0xda, 0x06, 0x5d, 0x12, 0xdd, + 0xfe, 0x2e, 0x32, 0xb5, 0x25, 0x15, 0xad, 0x8f, 0xbf, 0x18, 0x74, 0x9b, 0x7b, 0x5c, 0xd4, 0x27, + 0xa3, 0x74, 0x9e, 0x55, 0x92, 0x9e, 0xdc, 0x17, 0x2d, 0x6e, 0xab, 0xe8, 0xfc, 0xa8, 0x02, 0x1b, + 0x4b, 0x8f, 0xc2, 0xd3, 0x8c, 0x13, 0x18, 0x24, 0x45, 0xa9, 0x76, 0x36, 0x51, 0x6b, 0xd7, 0x13, + 0xa7, 0xfd, 0x0b, 0xce, 0xf9, 0x24, 0xf2, 0xbd, 0x96, 0xba, 0x90, 0xcd, 0xc3, 0x5b, 0xec, 0xca, + 0x7c, 0x58, 0xad, 0x4a, 0x59, 0x55, 0xb2, 0xa2, 0x02, 0xfa, 0xe2, 0x4c, 0x8c, 0x3f, 0xa0, 0x77, + 0xca, 0xef, 0x6e, 0x49, 0xb3, 0x6d, 0x2d, 0x93, 0xcc, 0x36, 0x1f, 0xa8, 0xc5, 0x38, 0x00, 0x36, + 0xa7, 0xa8, 0x43, 0xc1, 0xe6, 0x18, 0xc1, 0xbf, 0xa3, 0xff, 0xa2, 0xce, 0x9f, 0xe5, 0xc1, 0xc5, + 0xf2, 0x48, 0x70, 0x4d, 0x5f, 0xba, 0xf3, 0x2b, 0xf7, 0xdc, 0xa7, 0x5f, 0xe6, 0xee, 0x3b, 0x00, + 0x00, 0xff, 0xff, 0x95, 0x9e, 0x00, 0x7a, 0x3e, 0x02, 0x00, 0x00, } diff --git a/bchain/tx.proto b/bchain/tx.proto index 731b8bf5..61e06aed 100644 --- a/bchain/tx.proto +++ b/bchain/tx.proto @@ -11,7 +11,7 @@ syntax = "proto3"; repeated string Addresses = 6; } message VoutType { - double Value = 1; + bytes ValueSat = 1; uint32 N = 2; bytes ScriptPubKeyHex = 3; repeated string Addresses = 4; diff --git a/bchain/types.go b/bchain/types.go index e2aa7276..758f15f9 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math/big" ) // errors with specific meaning returned by blockchain rpc @@ -49,7 +50,8 @@ type Address interface { } type Vout struct { - Value float64 `json:"value"` + ValueSat big.Int + JsonValue json.Number `json:"value"` N uint32 `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` Address Address @@ -90,9 +92,9 @@ type BlockHeader struct { type MempoolEntry struct { Size uint32 `json:"size"` - Fee float64 `json:"fee"` - ModifiedFee float64 `json:"modifiedfee"` - Time float64 `json:"time"` + Fee big.Int `json:"fee"` + ModifiedFee big.Int `json:"modifiedfee"` + Time uint64 `json:"time"` Height uint32 `json:"height"` DescendantCount uint32 `json:"descendantcount"` DescendantSize uint32 `json:"descendantsize"` @@ -132,8 +134,8 @@ type BlockChain interface { GetMempool() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) - EstimateSmartFee(blocks int, conservative bool) (float64, error) - EstimateFee(blocks int) (float64, error) + EstimateSmartFee(blocks int, conservative bool) (big.Int, error) + EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) // mempool ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) @@ -145,7 +147,7 @@ type BlockChain interface { // BlockChainParser defines common interface to parsing and conversions of block chain data type BlockChainParser interface { - // self description + // chain configuration description // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db IsUTXOChain() bool @@ -153,6 +155,11 @@ type BlockChainParser interface { // and used in case of fork // if 0 the blockaddresses column is not used at all (usually non UTXO chains) KeepBlockAddresses() int + // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place + AmountToDecimalString(a *big.Int) string + // AmountToBigInt converts amount in json.Number (string) to big.Int + // it uses string operations to avoid problems with rounding + AmountToBigInt(n json.Number) (big.Int, error) // address id conversions GetAddrIDFromVout(output *Vout) ([]byte, error) GetAddrIDFromAddress(address string) ([]byte, error) diff --git a/server/public.go b/server/public.go index 54555325..acafd08c 100644 --- a/server/public.go +++ b/server/public.go @@ -113,8 +113,10 @@ func formatUnixTime(ut int64) string { return time.Unix(ut, 0).Format(time.RFC1123) } -func formatAmount(a float64) string { - return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%0.8f", a), "0"), ".") +// for now return the string as it is +// in future could be used to do coin specific formatting +func formatAmount(a string) string { + return a } // Run starts the server diff --git a/server/socketio.go b/server/socketio.go index 46c0a563..74fbd67b 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -7,6 +7,7 @@ import ( "blockbook/db" "encoding/json" "net/http" + "strconv" "strings" "time" @@ -236,11 +237,11 @@ type txInputs struct { // ScriptAsm *string `json:"scriptAsm"` Sequence int64 `json:"sequence"` Address *string `json:"address"` - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` } type txOutputs struct { - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` Script *string `json:"script"` // ScriptAsm *string `json:"scriptAsm"` SpentTxID *string `json:"spentTxId,omitempty"` @@ -267,7 +268,7 @@ type resTx struct { type addressHistoryItem struct { Addresses map[string]addressHistoryIndexes `json:"addresses"` - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` Confirmations int `json:"confirmations"` Tx resTx `json:"tx"` } @@ -329,7 +330,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r for _, vout := range tx.Vout { aoh := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.String(), Script: &aoh, } if vout.Address != nil { @@ -452,6 +453,7 @@ func unmarshalEstimateSmartFee(params []byte) (blocks int, conservative bool, er } type resultEstimateSmartFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -460,7 +462,7 @@ func (s *SocketIoServer) estimateSmartFee(blocks int, conservative bool) (res re if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -479,6 +481,7 @@ func unmarshalEstimateFee(params []byte) (blocks int, err error) { } type resultEstimateFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -487,7 +490,7 @@ func (s *SocketIoServer) estimateFee(blocks int) (res resultEstimateFee, err err if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -588,7 +591,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai a := vout.Address.String() ai.Address = &a } - ai.Satoshis = int64(vout.Value*1E8 + 0.5) + ai.Satoshis = vout.ValueSat.String() } } hi = append(hi, ai) @@ -596,7 +599,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai for _, vout := range tx.Vout { aos := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.String(), Script: &aos, } if vout.Address != nil { From 1ae62cc974503b21dbae66f046f10a05fe06df63 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 24 Jul 2018 21:21:05 +0200 Subject: [PATCH 002/123] Fix coin parser tests after switch to big.Int amounts --- bchain/coins/bch/bcashparser_test.go | 13 +- bchain/coins/btc/bitcoinparser_test.go | 13 +- bchain/coins/dogecoin/dogecoinparser_test.go | 17 +- bchain/coins/eth/ethparser_test.go | 173 +++++++++---------- bchain/coins/litecoin/litecoinparser_test.go | 9 +- bchain/coins/vertcoin/vertcoinparser_test.go | 9 +- bchain/coins/zec/zcashparser_test.go | 17 +- 7 files changed, 119 insertions(+), 132 deletions(-) diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index 3e1cb4c1..d6040b82 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -7,6 +7,7 @@ import ( "blockbook/bchain/coins/btc" "bytes" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -159,8 +160,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ @@ -190,8 +191,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ @@ -201,8 +202,8 @@ func init() { Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index d605f45b..79d02816 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -5,6 +5,7 @@ package btc import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -152,8 +153,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ @@ -183,8 +184,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ @@ -194,8 +195,8 @@ func init() { Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ diff --git a/bchain/coins/dogecoin/dogecoinparser_test.go b/bchain/coins/dogecoin/dogecoinparser_test.go index b8ea585f..0b736c51 100644 --- a/bchain/coins/dogecoin/dogecoinparser_test.go +++ b/bchain/coins/dogecoin/dogecoinparser_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "path/filepath" "reflect" "testing" @@ -111,8 +112,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 27478.75452951, - N: 0, + ValueSat: *big.NewInt(2747875452951), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914eef21768a546590993e313c7f3dfadf6a6efa1e888ac", Addresses: []string{ @@ -122,8 +123,8 @@ func init() { Address: addr1, }, { - Value: 74.20567469, - N: 1, + ValueSat: *big.NewInt(7420567469), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e0fee2ea29dd9c6c759d8341bd0da4c4f738cced88ac", Addresses: []string{ @@ -153,8 +154,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 59890867.89818935, - N: 0, + ValueSat: *big.NewInt(5989086789818935), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149355c01ed20057eac9fe0bbf8b07d87e62fe712d88ac", Addresses: []string{ @@ -164,8 +165,8 @@ func init() { Address: addr3, }, { - Value: 9999998.90000000, - N: 1, + ValueSat: *big.NewInt(999999890000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9145b4f2511c94e4fcaa8f8835b2458f8cb6542ca7688ac", Addresses: []string{ diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index c1f4620f..40069a55 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -5,6 +5,7 @@ package eth import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -63,6 +64,68 @@ func TestEthParser_GetAddrIDFromAddress(t *testing.T) { } } +var ( + testTx1, testTx2 bchain.Tx + testTxPacked1 = "08aebf0a1205012a05f20018a0f73622081234567890abcdef2a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f" + testTxPacked2 = "08ece40212050430e234001888a4012201213220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915" +) + +func init() { + var ( + addr1, addr2 bchain.Address + err error + ) + addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") + if err == nil { + addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") + } + if err != nil { + panic(err) + } + + testTx1 = bchain.Tx{ + Blocktime: 1521515026, + Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307831323334353637383930616263646566222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", + Time: 1521515026, + Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + Vin: []bchain.Vin{ + { + Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(1311768467294899695), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, + }, + Address: addr1, + }, + }, + } + + testTx2 = bchain.Tx{ + Blocktime: 1521533434, + Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a2230783231222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", + Time: 1521533434, + Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", + Vin: []bchain.Vin{ + { + Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(33), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, + }, + Address: addr2, + }, + }, + } +} + func TestEthereumParser_PackTx(t *testing.T) { type args struct { tx *bchain.Tx @@ -77,61 +140,27 @@ func TestEthereumParser_PackTx(t *testing.T) { wantErr bool }{ { - name: "with 0x prefix", + name: "1", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - }, - }, - }, + tx: &testTx1, height: 2870000, blockTime: 1521515026, }, - want: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f", + want: testTxPacked1, }, { - name: "without 0x prefix", + name: "2", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - }, - }, - }, + tx: &testTx2, height: 2871048, blockTime: 1521533434, }, - want: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915", + want: testTxPacked2, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime) if (err != nil) != tt.wantErr { t.Errorf("EthereumParser.PackTx() error = %v, wantErr %v", err, tt.wantErr) @@ -146,18 +175,6 @@ func TestEthereumParser_PackTx(t *testing.T) { } func TestEthereumParser_UnpackTx(t *testing.T) { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") - if err == nil { - addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") - } - if err != nil { - panic(err) - } - type args struct { hex string } @@ -170,57 +187,21 @@ func TestEthereumParser_UnpackTx(t *testing.T) { wantErr bool }{ { - name: "1", - args: args{hex: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f"}, - want: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - Address: addr1, - }, - }, - }, + name: "1", + args: args{hex: testTxPacked1}, + want: &testTx1, want1: 2870000, }, { - name: "1", - args: args{hex: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915"}, - want: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - Address: addr2, - }, - }, - }, + name: "2", + args: args{hex: testTxPacked2}, + want: &testTx2, want1: 2871048, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() b, err := hex.DecodeString(tt.args.hex) if err != nil { panic(err) diff --git a/bchain/coins/litecoin/litecoinparser_test.go b/bchain/coins/litecoin/litecoinparser_test.go index 11c55f1e..a76faf63 100644 --- a/bchain/coins/litecoin/litecoinparser_test.go +++ b/bchain/coins/litecoin/litecoinparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -173,8 +174,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 12.52000000, - N: 0, + ValueSat: *big.NewInt(1252000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9141ae882e788091732da6910595314447c9e38bd8d88ac", Addresses: []string{ @@ -184,8 +185,8 @@ func init() { Address: addr1, }, { - Value: 0.01000487, - N: 1, + ValueSat: *big.NewInt(1000487), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9146b474cbf0f6004329b630bdd4798f2c23d1751b688ac", Addresses: []string{ diff --git a/bchain/coins/vertcoin/vertcoinparser_test.go b/bchain/coins/vertcoin/vertcoinparser_test.go index 558ae51f..0cbf01d5 100644 --- a/bchain/coins/vertcoin/vertcoinparser_test.go +++ b/bchain/coins/vertcoin/vertcoinparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -105,8 +106,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.10781192, - N: 0, + ValueSat: *big.NewInt(10781192), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a91499b16da88a7e29b913b6131df2644d6d06cb331b88ac", Addresses: []string{ @@ -116,8 +117,8 @@ func init() { Address: addr1, }, { - Value: 0.5000000, - N: 1, + ValueSat: *big.NewInt(50000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a91446eb90e002f137f05385896c882fe000cc2e967f87", Addresses: []string{ diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index 0bb61fe1..b1bfd773 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -13,8 +14,8 @@ import ( var ( testTx1, testTx2 bchain.Tx - testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a4b091e6c90cd3ebc664010001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" - testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a4b09257b2170264d504010001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a4b099a9999999999b93f10011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" + testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a490a05043c1aec8e10001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" + testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a490a050184a7a72310001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a470a0398968010011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" ) func init() { @@ -51,8 +52,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 181.88266638, - N: 0, + ValueSat: *big.NewInt(18188266638), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac", Addresses: []string{ @@ -82,8 +83,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 65.20547107, - N: 0, + ValueSat: *big.NewInt(6520547107), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914826f87806ddd4643730be99b41c98acc379e83db88ac", Addresses: []string{ @@ -93,8 +94,8 @@ func init() { Address: addr2, }, { - Value: .1, - N: 1, + ValueSat: *big.NewInt(10000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e395634b7684289285926d4c64db395b783720ec88ac", Addresses: []string{ From f80a9142d9becee2a884323e1a49e48c3804c446 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 25 Jul 2018 11:47:07 +0200 Subject: [PATCH 003/123] Handle negative big.Int amounts, add tests of amounts parser --- bchain/baseparser.go | 21 +++++++++------ bchain/baseparser_test.go | 56 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 bchain/baseparser_test.go diff --git a/bchain/baseparser.go b/bchain/baseparser.go index d8d04fef..ce6dc947 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -65,18 +65,23 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place func (p *BaseParser) AmountToDecimalString(a *big.Int) string { - s := a.String() - if len(s) <= p.AmountDecimalPoint { - s = zeros[:p.AmountDecimalPoint-len(s)+1] + s + n := a.String() + var s string + if n[0] == '-' { + n = n[1:] + s = "-" } - i := len(s) - p.AmountDecimalPoint - ad := strings.TrimRight(s[i:], "0") + if len(n) <= p.AmountDecimalPoint { + n = zeros[:p.AmountDecimalPoint-len(n)+1] + n + } + i := len(n) - p.AmountDecimalPoint + ad := strings.TrimRight(n[i:], "0") if len(ad) > 0 { - s = s[:i] + "." + ad + n = n[:i] + "." + ad } else { - s = s[:i] + n = n[:i] } - return s + return s + n } // ParseTxFromJson parses JSON message containing transaction and returns Tx struct diff --git a/bchain/baseparser_test.go b/bchain/baseparser_test.go new file mode 100644 index 00000000..cfb20a68 --- /dev/null +++ b/bchain/baseparser_test.go @@ -0,0 +1,56 @@ +package bchain + +import ( + "encoding/json" + "math/big" + "testing" +) + +func NewBaseParser(adp int) *BaseParser { + return &BaseParser{ + AmountDecimalPoint: adp, + } +} + +var amounts = []struct { + a *big.Int + s string + adp int + alternative string +}{ + {big.NewInt(123456789), "1.23456789", 8, "!"}, + {big.NewInt(2), "0.00000002", 8, "!"}, + {big.NewInt(300000000), "3", 8, "!"}, + {big.NewInt(498700000), "4.987", 8, "!"}, + {big.NewInt(567890), "0.00000000000056789", 18, "!"}, + {big.NewInt(-100000000), "-1", 8, "!"}, + {big.NewInt(-8), "-0.00000008", 8, "!"}, + {big.NewInt(-89012345678), "-890.12345678", 8, "!"}, + {big.NewInt(-12345), "-0.00012345", 8, "!"}, + {big.NewInt(12345678), "0.123456789012", 8, "0.12345678"}, // test of truncation of too many decimal places +} + +func TestBaseParser_AmountToDecimalString(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + if got := NewBaseParser(tt.adp).AmountToDecimalString(tt.a); got != tt.s && got != tt.alternative { + t.Errorf("BaseParser.AmountToDecimalString() = %v, want %v", got, tt.s) + } + }) + } +} + +func TestBaseParser_AmountToBigInt(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + got, err := NewBaseParser(tt.adp).AmountToBigInt(json.Number(tt.s)) + if err != nil { + t.Errorf("BaseParser.AmountToBigInt() error = %v", err) + return + } + if got.Cmp(tt.a) != 0 { + t.Errorf("BaseParser.AmountToBigInt() = %v, want %v", got, tt.a) + } + }) + } +} From 882ff1bfd5adedafbfed89d388debd06aa94282a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 25 Jul 2018 15:56:08 +0200 Subject: [PATCH 004/123] Fix integration tests after switch to big.Int amounts --- bchain/baseparser.go | 2 ++ bchain/coins/bch/bcashparser.go | 1 + bchain/coins/btc/bitcoinrpc.go | 9 +++++- bchain/coins/zec/zcashparser.go | 3 +- bchain/tests/rpc/data.go | 16 ++++++++-- bchain/tests/rpc/rpc.go | 28 +++++++++++------- bchain/tests/rpc/testdata/Bcash.json | 2 ++ bchain/tests/rpc/testdata/Bcash_Testnet.json | 2 ++ bchain/tests/rpc/testdata/Bitcoin.json | 2 ++ .../tests/rpc/testdata/Bitcoin_Testnet.json | 2 ++ bchain/tests/rpc/testdata/Dash.json | 1 + bchain/tests/rpc/testdata/Dash_Testnet.json | 2 ++ .../tests/rpc/testdata/Ethereum_Testnet.json | 29 ++++--------------- bchain/tests/rpc/testdata/Namecoin.json | 1 + bchain/tests/rpc/testdata/Vertcoin.json | 2 ++ bchain/tests/rpc/testdata/Zcash.json | 2 ++ bchain/tests/rpc/testdata/Zcash_Testnet.json | 2 ++ bchain/types.go | 26 +++++++++-------- 18 files changed, 81 insertions(+), 51 deletions(-) diff --git a/bchain/baseparser.go b/bchain/baseparser.go index ce6dc947..172a8180 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -94,10 +94,12 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { for i := range tx.Vout { vout := &tx.Vout[i] + // convert vout.JsonValue to big.Int and clear it, it is only temporary value used for unmarshal vout.ValueSat, err = p.AmountToBigInt(vout.JsonValue) if err != nil { return nil, err } + vout.JsonValue = "" if len(vout.ScriptPubKey.Addresses) == 1 { a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) if err != nil { diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 95284d84..02286ca6 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -49,6 +49,7 @@ func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) (*BCashParser BaseParser: &bchain.BaseParser{ AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, format) }, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, Params: params, OutputScriptToAddressesFunc: outputScriptToAddresses, diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index 8f9fb067..c8a9f476 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -710,13 +710,20 @@ func (b *BitcoinRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) Params: []string{txid}, } err := b.Call(&req, &res) - if err != nil { return nil, err } if res.Error != nil { return nil, res.Error } + res.Result.FeeSat, err = b.Parser.AmountToBigInt(res.Result.Fee) + if err != nil { + return nil, err + } + res.Result.ModifiedFeeSat, err = b.Parser.AmountToBigInt(res.Result.ModifiedFee) + if err != nil { + return nil, err + } return res.Result, nil } diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index eacf8bbd..dc2d84d9 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -20,12 +20,13 @@ type ZCashParser struct { *bchain.BaseParser } -// NewZCAshParser returns new ZCAshParser instance +// NewZCashParser returns new ZCashParser instance func NewZCashParser(c *btc.Configuration) *ZCashParser { return &ZCashParser{ &bchain.BaseParser{ AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, } } diff --git a/bchain/tests/rpc/data.go b/bchain/tests/rpc/data.go index 7d50460a..68d69acf 100644 --- a/bchain/tests/rpc/data.go +++ b/bchain/tests/rpc/data.go @@ -1,8 +1,7 @@ -// +build integration - package rpc import ( + "blockbook/bchain" "encoding/json" "errors" "fmt" @@ -83,7 +82,7 @@ func LoadRPCConfig(coin string) (json.RawMessage, error) { return json.RawMessage(fmt.Sprintf(t, coin, c.URL, c.User, c.Pass)), nil } -func LoadTestData(coin string) (*TestData, error) { +func LoadTestData(coin string, parser bchain.BlockChainParser) (*TestData, error) { b, err := readDataFile(".", "bchain/tests/rpc/testdata", coin+".json") if err != nil { return nil, err @@ -93,5 +92,16 @@ func LoadTestData(coin string) (*TestData, error) { if err != nil { return nil, err } + // convert amounts in test json to bit.Int and clear the temporary JsonValue + for _, tx := range v.TxDetails { + for i := range tx.Vout { + vout := &tx.Vout[i] + vout.ValueSat, err = parser.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } + vout.JsonValue = "" + } + } return &v, nil } diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go index 4386baa1..20483573 100644 --- a/bchain/tests/rpc/rpc.go +++ b/bchain/tests/rpc/rpc.go @@ -1,5 +1,3 @@ -// +build integration - package rpc import ( @@ -9,6 +7,7 @@ import ( "net" "reflect" "testing" + "time" "github.com/deckarep/golang-set" ) @@ -56,7 +55,7 @@ func NewTest(coin string, factory TestChainFactoryFunc) (*Test, error) { return nil, err } } else { - td, err = LoadTestData(coin) + td, err = LoadTestData(coin, cli.GetChainParser()) if err != nil { return nil, err } @@ -162,7 +161,7 @@ func (rt *Test) TestGetTransaction(t *testing.T) { got.Confirmations = 0 if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransaction() got %v, want %v", got, want) + t.Errorf("GetTransaction() got %+v, want %+v", got, want) } } } @@ -181,7 +180,7 @@ func (rt *Test) TestGetTransactionForMempool(t *testing.T) { // transactions parsed from JSON may contain additional data got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransactionForMempool() got %v, want %v", got, want) + t.Errorf("GetTransactionForMempool() got %+v, want %+v", got, want) } } } @@ -329,8 +328,8 @@ func (rt *Test) TestGetMempoolEntry(t *testing.T) { if e.Size <= 0 { t.Errorf("GetMempoolEntry() got zero or negative size %d", e.Size) } - if e.Fee <= 0 { - t.Errorf("GetMempoolEntry() got zero or negative fee %f", e.Fee) + if e.FeeSat.Sign() != 1 { + t.Errorf("GetMempoolEntry() got zero or negative fee %v", e.FeeSat.String()) } // done @@ -347,8 +346,11 @@ func (rt *Test) TestEstimateSmartFee(t *testing.T) { if err != nil { t.Error(err) } - if fee != -1 && fee < 0 { - t.Errorf("EstimateSmartFee() returned unexpected fee rate: %f", fee) + if fee.Sign() == -1 { + sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateSmartFee() returned unexpected fee rate: %v", sf) + } } } } @@ -361,8 +363,11 @@ func (rt *Test) TestEstimateFee(t *testing.T) { if err != nil { t.Error(err) } - if fee != -1 && fee < 0 { - t.Errorf("EstimateFee() returned unexpected fee rate: %f", fee) + if fee.Sign() == -1 { + sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateFee() returned unexpected fee rate: %v", sf) + } } } } @@ -385,6 +390,7 @@ func (rt *Test) TestGetBestBlockHash(t *testing.T) { t.Fatal(err) } if hash != hh { + time.Sleep(time.Millisecond * 100) continue } diff --git a/bchain/tests/rpc/testdata/Bcash.json b/bchain/tests/rpc/testdata/Bcash.json index f72df31f..9b1f647b 100644 --- a/bchain/tests/rpc/testdata/Bcash.json +++ b/bchain/tests/rpc/testdata/Bcash.json @@ -70,6 +70,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "fe496933e0a3582ef020bd35c38fe8244a80fa7c63e7607f6f9ccb0806f419e4", @@ -175,6 +176,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "52f37874fd5a2497c5d84619ab415ca17ebb1d49d559f0d468869f70537354b9", diff --git a/bchain/tests/rpc/testdata/Bcash_Testnet.json b/bchain/tests/rpc/testdata/Bcash_Testnet.json index 39eba5e7..c33176d3 100644 --- a/bchain/tests/rpc/testdata/Bcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Bcash_Testnet.json @@ -12,6 +12,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "coinbase": "03fbf212055374617368", @@ -34,6 +35,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "txid": "58ff2450a71b3c228d17d04ec8edcaae452bc97d5ccc4591e2741fd8b031d221", diff --git a/bchain/tests/rpc/testdata/Bitcoin.json b/bchain/tests/rpc/testdata/Bitcoin.json index ba0467cc..5cf34ec8 100644 --- a/bchain/tests/rpc/testdata/Bitcoin.json +++ b/bchain/tests/rpc/testdata/Bitcoin.json @@ -120,6 +120,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 0, + "version": 1, "vin": [ { "txid": "83eff5bcc738d52e04528984cd5cf601b69e7df65b2b10ed2475c0072cdde14c", @@ -153,6 +154,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 529149, + "version": 2, "vin": [ { "txid": "413988d546516707f3b20ca9876e026a9472bd814662e42f37f4e57a15713da7", diff --git a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json index ece3c941..cab2c840 100644 --- a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json +++ b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json @@ -43,6 +43,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "4097f5265397047bffe219a1bca65ea5726402c3d9aeecd577fa1c274fc1b8a4", @@ -84,6 +85,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "80437ba4e81bde8b584258f5adad0d08dea60f1e38026344442ad59a4ef797c9", diff --git a/bchain/tests/rpc/testdata/Dash.json b/bchain/tests/rpc/testdata/Dash.json index 150e7d51..f5ab09b6 100644 --- a/bchain/tests/rpc/testdata/Dash.json +++ b/bchain/tests/rpc/testdata/Dash.json @@ -27,6 +27,7 @@ "blocktime": 1530189699, "time": 1530189699, "locktime": 0, + "version": 1, "vin": [ { "txid": "a78824a88c089ea1344fe2a33c454024e6501e76cca2590515b390d85082bd23", diff --git a/bchain/tests/rpc/testdata/Dash_Testnet.json b/bchain/tests/rpc/testdata/Dash_Testnet.json index d86f0bee..64509772 100644 --- a/bchain/tests/rpc/testdata/Dash_Testnet.json +++ b/bchain/tests/rpc/testdata/Dash_Testnet.json @@ -31,6 +31,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "a41616c7585c98aeda98d6ff6766b15455e327c9472582b80289dab7597ad309", @@ -64,6 +65,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "187ae015e41dff766f5a18ae705f59db950a6729a06fa5fd04630c362a9aee27", diff --git a/bchain/tests/rpc/testdata/Ethereum_Testnet.json b/bchain/tests/rpc/testdata/Ethereum_Testnet.json index 82bbf3c7..b7cd76cb 100644 --- a/bchain/tests/rpc/testdata/Ethereum_Testnet.json +++ b/bchain/tests/rpc/testdata/Ethereum_Testnet.json @@ -16,38 +16,21 @@ "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0" ], "txDetails": { - "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d": { - "hex": "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - "txid": "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0": { + "hex": "7b226e6f6e6365223a223078333632306136222c226761735072696365223a223078313261303566323030222c22676173223a22307835323038222c22746f223a22307831623137626331326166623635643563346238316139666632613037366234326131396661616136222c2276616c7565223a223078646530623662336137363430303030222c22696e707574223a223078222c2268617368223a22307837663064313430333239393431663132306235623366633735316533306164656238376232616562626663653561646364303231363630346133346236636330222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838316237653038663635626466353634383630366338393939386139636338313634333937363437222c227472616e73616374696f6e496e646578223a22307862222c2276223a2230783162222c2272223a22307862666662323864633865373939383833366639356664616139396433626435666635346365663562313839636462383537333537666161326431616231393136222c2273223a22307833616462316365313264396664306538616662373064386639346534636538356137666639346465333966623636333139363638363435663464643138646432227d", + "txid": "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0", "blocktime": 1521515026, "time": 1521515026, "vin": [ { - "addresses": ["0xdacc9c61754a0c4616fc5323dc946e89eb272302"] + "addresses": ["0x81b7e08f65bdf5648606c89998a9cc8164397647"] } ], "vout": [ { + "value": 1, "scriptPubKey": { - "addresses": ["0x682b7903a11098cf770c7aef4aa02a85b3f3601a"] - } - } - ] - }, - "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3": { - "hex": "7b226e6f6e6365223a223078346566222c226761735072696365223a223078626134336237343030222c22676173223a2230783936653838222c22746f223a22307864303534323939633438326164356362333961336661383734373235663965326132306433343963222c2276616c7565223a22307830222c22696e707574223a2230783165626366613630303030303030303030303030303030303030303030303030633034383732383566313736663532306635636434363732306236383662366534366165636430373030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303031303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030313430303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030633461303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303530303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030353831303030303030303030303030303030303030303030303030633365313631343261306432366336616530623163656238643961373236653734653164613664663666656562653137653437306464626434656261663364623938333136386235316337623136363732613131623966306662383864623438323962653237346230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303034363336663665363230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303437323666373336313030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030222c2268617368223a22307831376565323335666330333539313535623235343139653065346336356439633530306466366537316538323838643665663032306430346363326632636233222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838623461353365353739303739343466633261653539623837373066393331623639326232373062222c227472616e73616374696f6e496e646578223a22307830222c2276223a2230783163222c2272223a22307866633739633836353638323039313030323134616339353662373930653066383935656130343135313438643233613239353632356564393761633936333534222c2273223a223078373232303833616331643764663662626162393939383537616163323561336665646265386130633561326266376364653835393738363266373862313937227d", - "txid": "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", - "blocktime": 1521515026, - "time": 1521515026, - "vin": [ - { - "addresses": ["0x8b4a53e57907944fc2ae59b8770f931b692b270b"] - } - ], - "vout": [ - { - "scriptPubKey": { - "addresses": ["0xd054299c482ad5cb39a3fa874725f9e2a20d349c"] + "addresses": ["0x1b17bc12afb65d5c4b81a9ff2a076b42a19faaa6"] } } ] diff --git a/bchain/tests/rpc/testdata/Namecoin.json b/bchain/tests/rpc/testdata/Namecoin.json index 4e1193cd..5a56c972 100644 --- a/bchain/tests/rpc/testdata/Namecoin.json +++ b/bchain/tests/rpc/testdata/Namecoin.json @@ -39,6 +39,7 @@ "blocktime": 1530003649, "time": 1530003649, "locktime": 404678, + "version": 1, "vin": [ { "txid": "223a4ff7a613173ffbc67f9efffe1b0c3fd5ba2471935a44cc81814038272901", diff --git a/bchain/tests/rpc/testdata/Vertcoin.json b/bchain/tests/rpc/testdata/Vertcoin.json index 021cdc2a..d5c5383e 100644 --- a/bchain/tests/rpc/testdata/Vertcoin.json +++ b/bchain/tests/rpc/testdata/Vertcoin.json @@ -14,6 +14,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952232, + "version": 2, "vin": [ { "txid": "8ec70404bdd16559c589736863ee9d9f69431ba4e6b2ab3b1641b3f9f3821624", @@ -47,6 +48,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952233, + "version": 1, "vin": [ { "txid": "5765db7dd62cddada154b2c95505e298818a308cc1ce1e28d6e8565301cb8d74", diff --git a/bchain/tests/rpc/testdata/Zcash.json b/bchain/tests/rpc/testdata/Zcash.json index 6b620537..346d9f96 100644 --- a/bchain/tests/rpc/testdata/Zcash.json +++ b/bchain/tests/rpc/testdata/Zcash.json @@ -22,6 +22,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 349399, + "version": 3, "vin": [ { "txid": "9c9faac29b0fa1b0e683727f2973bfcb87e4baadd07e9c997b431a31713cb30c", @@ -57,6 +58,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 0, + "version": 3, "vin": [ { "txid": "4440c8e9d5b57da7ca0fb3f62ec13f392267aeb797c123bb01e850adf8573dd0", diff --git a/bchain/tests/rpc/testdata/Zcash_Testnet.json b/bchain/tests/rpc/testdata/Zcash_Testnet.json index aca84131..5d78989c 100644 --- a/bchain/tests/rpc/testdata/Zcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Zcash_Testnet.json @@ -13,6 +13,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251028, + "version": 3, "vin": [ { "txid": "19a1d013b898239e9a2943faa07f8716b9be168bc8e001daf3625f535fde1a60", @@ -48,6 +49,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251090, + "version": 3, "vin": [ { "txid": "9acab5f13cf94074e75f5686b59fccd938f54b5f20ddddfcb6077c679a13c0ea", diff --git a/bchain/types.go b/bchain/types.go index 758f15f9..cfe90771 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -91,18 +91,20 @@ type BlockHeader struct { } type MempoolEntry struct { - Size uint32 `json:"size"` - Fee big.Int `json:"fee"` - ModifiedFee big.Int `json:"modifiedfee"` - Time uint64 `json:"time"` - Height uint32 `json:"height"` - DescendantCount uint32 `json:"descendantcount"` - DescendantSize uint32 `json:"descendantsize"` - DescendantFees uint32 `json:"descendantfees"` - AncestorCount uint32 `json:"ancestorcount"` - AncestorSize uint32 `json:"ancestorsize"` - AncestorFees uint32 `json:"ancestorfees"` - Depends []string `json:"depends"` + Size uint32 `json:"size"` + FeeSat big.Int + Fee json.Number `json:"fee"` + ModifiedFeeSat big.Int + ModifiedFee json.Number `json:"modifiedfee"` + Time uint64 `json:"time"` + Height uint32 `json:"height"` + DescendantCount uint32 `json:"descendantcount"` + DescendantSize uint32 `json:"descendantsize"` + DescendantFees uint32 `json:"descendantfees"` + AncestorCount uint32 `json:"ancestorcount"` + AncestorSize uint32 `json:"ancestorsize"` + AncestorFees uint32 `json:"ancestorfees"` + Depends []string `json:"depends"` } type RPCError struct { From d6cbe7c513623f7e56851ab3e799e960fa2799f0 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 26 Jul 2018 00:15:29 +0200 Subject: [PATCH 005/123] Remove duplicate mempool transactions --- api/worker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/worker.go b/api/worker.go index ea2870e6..01064560 100644 --- a/api/worker.go +++ b/api/worker.go @@ -183,6 +183,7 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { if err != nil { return nil, err } + txm = UniqueTxidsInReverse(txm) bestheight, _, err := w.db.GetBestBlock() if err != nil { return nil, err From 75d48376e14f6fb19b1af3e80759518ee86183e9 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 27 Jul 2018 14:08:20 +0200 Subject: [PATCH 006/123] Change db columns, enable compression on some columns --- db/rocksdb.go | 73 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 30285dab..885b57d1 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -46,12 +46,17 @@ const ( cfDefault = iota cfHeight cfAddresses - cfUnspentTxs + cfTxAddresses + cfAddressBalance + cfBlockTxids cfTransactions + + // to be removed, kept temporarily so that the code is compilable + cfUnspentTxs cfBlockAddresses ) -var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"} +var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -61,37 +66,49 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) bbto.SetBlockCache(c) bbto.SetFilterPolicy(fp) - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) - opts.SetMaxBackgroundCompactions(4) - opts.SetMaxBackgroundFlushes(2) - opts.SetBytesPerSync(1 << 20) // 1MB - opts.SetWriteBufferSize(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) - opts.SetCompression(gorocksdb.NoCompression) + optsNoCompression := gorocksdb.NewDefaultOptions() + optsNoCompression.SetBlockBasedTableFactory(bbto) + optsNoCompression.SetCreateIfMissing(true) + optsNoCompression.SetCreateIfMissingColumnFamilies(true) + optsNoCompression.SetMaxBackgroundCompactions(4) + optsNoCompression.SetMaxBackgroundFlushes(2) + optsNoCompression.SetBytesPerSync(1 << 20) // 1MB + optsNoCompression.SetWriteBufferSize(1 << 27) // 128MB + optsNoCompression.SetMaxOpenFiles(25000) + optsNoCompression.SetCompression(gorocksdb.NoCompression) - // opts for outputs are different: + optsLZ4 := gorocksdb.NewDefaultOptions() + optsLZ4.SetBlockBasedTableFactory(bbto) + optsLZ4.SetCreateIfMissing(true) + optsLZ4.SetCreateIfMissingColumnFamilies(true) + optsLZ4.SetMaxBackgroundCompactions(4) + optsLZ4.SetMaxBackgroundFlushes(2) + optsLZ4.SetBytesPerSync(1 << 20) // 1MB + optsLZ4.SetWriteBufferSize(1 << 27) // 128MB + optsLZ4.SetMaxOpenFiles(25000) + optsLZ4.SetCompression(gorocksdb.LZ4Compression) + + // opts for addresses are different: // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter - bbtoOutputs := gorocksdb.NewDefaultBlockBasedTableOptions() - bbtoOutputs.SetBlockSize(16 << 10) // 16kB - bbtoOutputs.SetBlockCache(c) // 8GB + bbtoAddresses := gorocksdb.NewDefaultBlockBasedTableOptions() + bbtoAddresses.SetBlockSize(16 << 10) // 16kB + bbtoAddresses.SetBlockCache(c) // 8GB - optsOutputs := gorocksdb.NewDefaultOptions() - optsOutputs.SetBlockBasedTableFactory(bbtoOutputs) - optsOutputs.SetCreateIfMissing(true) - optsOutputs.SetCreateIfMissingColumnFamilies(true) - optsOutputs.SetMaxBackgroundCompactions(4) - optsOutputs.SetMaxBackgroundFlushes(2) - optsOutputs.SetBytesPerSync(1 << 20) // 1MB - optsOutputs.SetWriteBufferSize(1 << 27) // 128MB - optsOutputs.SetMaxOpenFiles(25000) - optsOutputs.SetCompression(gorocksdb.NoCompression) + optsAddresses := gorocksdb.NewDefaultOptions() + optsAddresses.SetBlockBasedTableFactory(bbtoAddresses) + optsAddresses.SetCreateIfMissing(true) + optsAddresses.SetCreateIfMissingColumnFamilies(true) + optsAddresses.SetMaxBackgroundCompactions(4) + optsAddresses.SetMaxBackgroundFlushes(2) + optsAddresses.SetBytesPerSync(1 << 20) // 1MB + optsAddresses.SetWriteBufferSize(1 << 27) // 128MB + optsAddresses.SetMaxOpenFiles(25000) + optsAddresses.SetCompression(gorocksdb.NoCompression) - fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts} + // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions + fcOptions := []*gorocksdb.Options{optsNoCompression, optsNoCompression, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) + db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions) if err != nil { return nil, nil, err } From e558c10da927c8ccb99d10093bcec1078fc8b778 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 27 Jul 2018 19:46:21 +0200 Subject: [PATCH 007/123] Implement pack and unpack of big.Int --- db/rocksdb.go | 60 ++++++++++++++++++++++++++++++ db/rocksdb_test.go | 93 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 885b57d1..127b0cd8 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -6,6 +6,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" "os" "path/filepath" "time" @@ -1072,3 +1073,62 @@ func unpackVarint(buf []byte) (int32, int) { i, ofs := vlq.Int(buf) return int32(i), ofs } + +const ( + // number of bits in a big.Word + wordBits = 32 << (uint64(^big.Word(0)) >> 63) + // number of bytes in a big.Word + wordBytes = wordBits / 8 + // max packed bigint words + maxPackedBigintWords = (256 - wordBytes) / wordBytes +) + +// big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int +// number of written bytes is returned +// limitation: bigints longer than 248 bytes are truncated to 248 bytes +// caution: buffer must be big enough to hold the packed big int, buffer 249 bytes big is always safe +func packBigint(bi *big.Int, buf []byte) int { + w := bi.Bits() + lw := len(w) + // zero returns only one byte - zero length + if lw == 0 { + buf[0] = 0 + return 1 + } + // pack the most significant word in a special way - skip leading zeros + w0 := w[lw-1] + fb := 8 + mask := big.Word(0xff) << (wordBits - 8) + for w0&mask == 0 { + fb-- + mask >>= 8 + } + for i := fb; i > 0; i-- { + buf[i] = byte(w0) + w0 >>= 8 + } + // if the big int is too big (> 2^1984), the number of bytes would not fit to 1 byte + // in this case, truncate the number, it is not expected to work with this big numbers as amounts + s := 0 + if lw > maxPackedBigintWords { + s = lw - maxPackedBigintWords + } + // pack the rest of the words in reverse order + for j := lw - 2; j >= s; j-- { + d := w[j] + for i := fb + wordBytes; i > fb; i-- { + buf[i] = byte(d) + d >>= 8 + } + fb += wordBytes + } + buf[0] = byte(fb) + return fb + 1 +} + +func unpackBigint(buf []byte) (big.Int, int) { + var r big.Int + l := int(buf[0]) + 1 + r.SetBytes(buf[1:l]) + return r, l +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 52be1c5d..f34ecefb 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -1,4 +1,4 @@ -// +build unittest +// build unittest package db @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "os" "reflect" "sort" @@ -723,3 +724,93 @@ func Test_unpackBlockAddresses(t *testing.T) { }) } } + +func Test_packBigint_unpackBigint(t *testing.T) { + bigbig1, _ := big.NewInt(0).SetString("123456789123456789012345", 10) + bigbig2, _ := big.NewInt(0).SetString("12345678912345678901234512389012345123456789123456789012345123456789123456789012345", 10) + bigbigbig := big.NewInt(0) + bigbigbig.Mul(bigbig2, bigbig2) + bigbigbig.Mul(bigbigbig, bigbigbig) + bigbigbig.Mul(bigbigbig, bigbigbig) + tests := []struct { + name string + bi *big.Int + buf []byte + toobiglen int + }{ + { + name: "0", + bi: big.NewInt(0), + buf: make([]byte, 249), + }, + { + name: "1", + bi: big.NewInt(1), + buf: make([]byte, 249), + }, + { + name: "54321", + bi: big.NewInt(54321), + buf: make([]byte, 249), + }, + { + name: "12345678", + bi: big.NewInt(12345678), + buf: make([]byte, 249), + }, + { + name: "123456789123456789", + bi: big.NewInt(123456789123456789), + buf: make([]byte, 249), + }, + { + name: "bigbig1", + bi: bigbig1, + buf: make([]byte, 249), + }, + { + name: "bigbig2", + bi: bigbig2, + buf: make([]byte, 249), + }, + { + name: "bigbigbig", + bi: bigbigbig, + buf: make([]byte, 249), + toobiglen: 242, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // packBigint + got := packBigint(tt.bi, tt.buf) + if tt.toobiglen == 0 { + // create buffer that we expect + bb := tt.bi.Bytes() + want := append([]byte(nil), byte(len(bb))) + want = append(want, bb...) + if got != len(want) { + t.Errorf("packBigint() = %v, want %v", got, len(want)) + } + for i := 0; i < got; i++ { + if tt.buf[i] != want[i] { + t.Errorf("packBigint() buf = %v, want %v", tt.buf[:got], want) + break + } + } + // unpackBigint + got1, got2 := unpackBigint(tt.buf) + if got2 != len(want) { + t.Errorf("unpackBigint() = %v, want %v", got2, len(want)) + } + if tt.bi.Cmp(&got1) != 0 { + t.Errorf("unpackBigint() = %v, want %v", got1, tt.bi) + } + } else { + if got != tt.toobiglen { + t.Errorf("packBigint() = %v, want toobiglen %v", got, tt.toobiglen) + } + } + }) + } +} From a2bbf3f9de636dc0ce360b7b8fa3c7066807663d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 Aug 2018 14:30:45 +0200 Subject: [PATCH 008/123] Implement and test connectBlock for index v3 --- db/rocksdb.go | 450 ++++++++++++++++++++++++++++++++++++++++----- db/rocksdb_test.go | 332 +++++++++++++++++++-------------- 2 files changed, 603 insertions(+), 179 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 127b0cd8..89c920e5 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -22,7 +22,8 @@ import ( // when doing huge scan, it is better to close it and reopen from time to time to free the resources const refreshIterator = 5000000 const packedHeightBytes = 4 -const dbVersion = 0 +const dbVersion = 3 +const maxAddrIDLen = 1024 // RepairRocksDB calls RocksDb db repair function func RepairRocksDB(name string) error { @@ -203,11 +204,11 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f for _, o := range outpoints { var vout uint32 var isOutput bool - if o.vout < 0 { - vout = uint32(^o.vout) + if o.index < 0 { + vout = uint32(^o.index) isOutput = false } else { - vout = uint32(o.vout) + vout = uint32(o.index) isOutput = true } tx, err := d.chainParser.UnpackTxid(o.btxID) @@ -272,18 +273,381 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { type outpoint struct { btxID []byte - vout int32 + index int32 } +type txAddress struct { + addrID []byte + spent bool + valueSat big.Int +} + +type txAddresses struct { + inputs []txAddress + outputs []txAddress +} + +type addrBalance struct { + txs uint32 + sentSat big.Int + balanceSat big.Int +} + +func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } + addresses := make(map[string][]outpoint) + blockTxids := make([][]byte, len(block.Txs)) + txAddressesMap := make(map[string]*txAddresses) + balances := make(map[string]*addrBalance) + // first process all outputs so that inputs can point to txs in this block + for txi, tx := range block.Txs { + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return err + } + blockTxids[txi] = btxID + ta := txAddresses{} + ta.outputs = make([]txAddress, len(tx.Vout)) + txAddressesMap[string(btxID)] = &ta + for i, output := range tx.Vout { + tao := &ta.outputs[i] + tao.valueSat = output.ValueSat + addrID, err := d.chainParser.GetAddrIDFromVout(&output) + if err != nil || len(addrID) == 0 || len(addrID) > maxAddrIDLen { + if err != nil { + // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + } + } else { + glog.Infof("rocksdb: block %d, skipping addrID of length %d", block.Height, len(addrID)) + } + continue + } + tao.addrID = addrID + strAddrID := string(addrID) + // check that the address was used already in this block + o, processed := addresses[strAddrID] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, btxID) + } + addresses[strAddrID] = append(o, outpoint{ + btxID: btxID, + index: int32(i), + }) + ab, e := balances[strAddrID] + if !e { + ab, err = d.getAddressBalance(addrID) + if err != nil { + return err + } + if ab == nil { + ab = &addrBalance{} + } + balances[strAddrID] = ab + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.txs++ + } + ab.balanceSat.Add(&ab.balanceSat, &output.ValueSat) + } + } + // process inputs + for txi, tx := range block.Txs { + spendingTxid := blockTxids[txi] + ta := txAddressesMap[string(spendingTxid)] + ta.inputs = make([]txAddress, len(tx.Vin)) + for i, input := range tx.Vin { + tai := &ta.inputs[i] + btxID, err := d.chainParser.PackTxid(input.Txid) + if err != nil { + // do not process inputs without input txid + if err == bchain.ErrTxidMissing { + continue + } + return err + } + stxID := string(btxID) + ita, e := txAddressesMap[stxID] + if !e { + ita, err = d.getTxAddresses(btxID) + if err != nil { + return err + } + if ita == nil { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v not found in txAddresses", block.Height, tx.Txid, input.Txid) + continue + } + txAddressesMap[stxID] = ita + } + if len(ita.outputs) <= int(input.Vout) { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } + ot := &ita.outputs[int(input.Vout)] + if ot.spent { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } + tai.addrID = ot.addrID + tai.valueSat = ot.valueSat + // mark the output tx as spent + ot.spent = true + strAddrID := string(ot.addrID) + // check that the address was used already in this block + o, processed := addresses[strAddrID] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, spendingTxid) + } + addresses[strAddrID] = append(o, outpoint{ + btxID: spendingTxid, + index: ^int32(i), + }) + ab, e := balances[strAddrID] + if !e { + ab, err = d.getAddressBalance(ot.addrID) + if err != nil { + return err + } + if ab == nil { + ab = &addrBalance{} + } + balances[strAddrID] = ab + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.txs++ + } + ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) + if ab.balanceSat.Sign() < 0 { + ad, err := d.chainParser.OutputScriptToAddresses(ot.addrID) + if err != nil { + glog.Warningf("rocksdb: unparsable address reached negative balance %v, resetting to 0. Parser error %v", ab.balanceSat.String(), err) + } else { + glog.Warningf("rocksdb: address %v reached negative balance %v, resetting to 0", ab.balanceSat.String(), ad) + } + ab.balanceSat.SetInt64(0) + } + ab.sentSat.Add(&ab.sentSat, &ot.valueSat) + } + } + if op == opInsert { + if err := d.storeAddresses(wb, block, addresses); err != nil { + return err + } + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxids(wb, block, blockTxids); err != nil { + return err + } + } + return nil +} + +func processedInTx(o []outpoint, btxID []byte) bool { + for _, op := range o { + if bytes.Equal(btxID, op.btxID) { + return true + } + } + return false +} + +func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, block *bchain.Block, addresses map[string][]outpoint) error { + for addrID, outpoints := range addresses { + ba := []byte(addrID) + key := packAddressKey(ba, block.Height) + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + } + return nil +} + +func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAddresses) error { + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) + for txID, ta := range am { + buf = buf[:0] + l := packVaruint(uint(len(ta.inputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.inputs { + buf = appendTxAddress(buf, varBuf, &ta.inputs[i]) + } + l = packVaruint(uint(len(ta.outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.outputs { + buf = appendTxAddress(buf, varBuf, &ta.outputs[i]) + } + wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) + } + return nil +} + +func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBalance) error { + // allocate buffer big enough for number of txs + 2 bigints + buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) + for addrID, ab := range abm { + l := packVaruint(uint(ab.txs), buf) + ll := packBigint(&ab.sentSat, buf[l:]) + l += ll + ll = packBigint(&ab.balanceSat, buf[l:]) + l += ll + wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + } + return nil +} + +func appendTxAddress(buf []byte, varBuf []byte, txa *txAddress) []byte { + la := len(txa.addrID) + if txa.spent { + la = ^la + } + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txa.addrID...) + l = packBigint(&txa.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bchain.Block, txids [][]byte) error { + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, pl*len(txids)) + i := 0 + for _, txid := range txids { + copy(buf[i:], txid) + i += pl + } + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockTxids], key, buf) + keep := d.chainParser.KeepBlockAddresses() + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh < block.Height; rh-- { + key = packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], key) + if err != nil { + return err + } + if val.Size() == 0 { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxids], key) + } + } + return nil +} + +func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 3 is minimum length of addrBalance - 1 byte txs, 1 byte sent, 1 byte balance + if len(buf) < 3 { + return nil, nil + } + txs, l := unpackVaruint(buf) + sentSat, sl := unpackBigint(buf[l:]) + balanceSat, _ := unpackBigint(buf[l+sl:]) + return &addrBalance{ + txs: uint32(txs), + sentSat: sentSat, + balanceSat: balanceSat, + }, nil +} + +func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 2 is minimum length of addrBalance - 1 byte inputs len, 1 byte outputs len + if len(buf) < 2 { + return nil, nil + } + ta := txAddresses{} + inputs, l := unpackVaruint(buf) + ta.inputs = make([]txAddress, inputs) + for i := uint(0); i < inputs; i++ { + l += unpackTxAddress(&ta.inputs[i], buf[l:]) + } + outputs, ll := unpackVaruint(buf[l:]) + l += ll + ta.outputs = make([]txAddress, outputs) + for i := uint(0); i < outputs; i++ { + l += unpackTxAddress(&ta.outputs[i], buf[l:]) + } + return &ta, nil +} + +func unpackTxAddress(ta *txAddress, buf []byte) int { + al, l := unpackVarint(buf) + if al < 0 { + ta.spent = true + al ^= al + } + ta.addrID = make([]byte, al) + copy(ta.addrID, buf[l:l+al]) + al += l + ta.valueSat, l = unpackBigint(buf[al:]) + return l + al +} + +func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { + buf := make([]byte, 0) + bvout := make([]byte, vlq.MaxLen32) + for _, o := range outpoints { + l := packVarint32(o.index, bvout) + buf = append(buf, []byte(o.btxID)...) + buf = append(buf, bvout[:l]...) + } + return buf +} + +func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + outpoints := make([]outpoint, 0) + for i := 0; i < len(buf); { + btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) + i += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[i:]) + i += voutLen + outpoints = append(outpoints, outpoint{ + btxID: btxID, + index: vout, + }) + } + return outpoints, nil +} + +//////////////////////// + func (d *RocksDB) packBlockAddress(addrID []byte, spentTxs map[string][]outpoint) []byte { vBuf := make([]byte, vlq.MaxLen32) - vl := packVarint(int32(len(addrID)), vBuf) + vl := packVarint(len(addrID), vBuf) blockAddress := append([]byte(nil), vBuf[:vl]...) blockAddress = append(blockAddress, addrID...) if spentTxs == nil { } else { addrUnspentTxs := spentTxs[string(addrID)] - vl = packVarint(int32(len(addrUnspentTxs)), vBuf) + vl = packVarint(len(addrUnspentTxs), vBuf) blockAddress = append(blockAddress, vBuf[:vl]...) buf := d.packOutpoints(addrUnspentTxs) blockAddress = append(blockAddress, buf...) @@ -342,7 +706,7 @@ func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records m strAddrID := string(addrID) records[strAddrID] = append(records[strAddrID], outpoint{ btxID: btxid, - vout: vout, + index: vout, }) if op == opDelete { // remove transactions from cache @@ -370,10 +734,10 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) txAddrs = append(txAddrs, make([]byte, vlq.MaxLen32+len(addrID)+remaining*32)...)[:len(txAddrs)] } // addrID is packed as number of bytes of the addrID + bytes of addrID + vout - lv := packVarint(int32(len(addrID)), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) + lv := packVarint(len(addrID), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) txAddrs = txAddrs[:len(txAddrs)+lv] txAddrs = append(txAddrs, addrID...) - lv = packVarint(int32(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) + lv = packVarint(int(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) txAddrs = txAddrs[:len(txAddrs)+lv] return txAddrs } @@ -399,7 +763,7 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, []byte) return nil, unspentAddrs } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { +func (d *RocksDB) writeAddressesUTXO_old(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { if op == opDelete { // block does not contain mapping tx-> input address, which is necessary to recreate // unspentTxs; therefore it is not possible to DisconnectBlocks this way @@ -563,36 +927,9 @@ func (d *RocksDB) unpackBlockAddresses(buf []byte) ([][]byte, [][]outpoint, erro return addresses, outpointsArray, nil } -func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { - buf := make([]byte, 0) - bvout := make([]byte, vlq.MaxLen32) - for _, o := range outpoints { - l := packVarint(o.vout, bvout) - buf = append(buf, []byte(o.btxID)...) - buf = append(buf, bvout[:l]...) - } - return buf -} - -func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - outpoints := make([]outpoint, 0) - for i := 0; i < len(buf); { - btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) - i += txidUnpackedLen - vout, voutLen := unpackVarint(buf[i:]) - i += voutLen - outpoints = append(outpoints, outpoint{ - btxID: btxID, - vout: vout, - }) - } - return outpoints, nil -} - func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { txidUnpackedLen := d.chainParser.PackedTxidLen() - n, p := unpackVarint(buf) + n, p := unpackVarint32(buf) outpoints := make([]outpoint, n) for i := int32(0); i < n; i++ { if p+txidUnpackedLen >= len(buf) { @@ -600,11 +937,11 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { } btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) p += txidUnpackedLen - vout, voutLen := unpackVarint(buf[p:]) + vout, voutLen := unpackVarint32(buf[p:]) p += voutLen outpoints[i] = outpoint{ btxID: btxID, - vout: vout, + index: vout, } } return outpoints, p, nil @@ -616,7 +953,7 @@ func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { return nil, err } bv := make([]byte, vlq.MaxLen32) - l := packVarint(vout, bv) + l := packVarint32(vout, bv) buf := make([]byte, 0, l+len(btxid)) buf = append(buf, btxid...) buf = append(buf, bv[:l]...) @@ -626,10 +963,12 @@ func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { func (d *RocksDB) unpackOutpoint(buf []byte) (string, int32, int) { txidUnpackedLen := d.chainParser.PackedTxidLen() txid, _ := d.chainParser.UnpackTxid(buf[:txidUnpackedLen]) - vout, o := unpackVarint(buf[txidUnpackedLen:]) + vout, o := unpackVarint32(buf[txidUnpackedLen:]) return txid, vout, txidUnpackedLen + o } +////////////////////////////////// + // Block index // GetBestBlock returns the block hash of the block with highest height in the db @@ -799,7 +1138,7 @@ func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { return err } } - txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.vout), 1) + txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.index), 1) unspentTxs[stxID] = txAddrs } // delete unspentTxs from this block @@ -1065,15 +1404,33 @@ func unpackUint(buf []byte) uint32 { return binary.BigEndian.Uint32(buf) } -func packVarint(i int32, buf []byte) int { +func packVarint32(i int32, buf []byte) int { return vlq.PutInt(buf, int64(i)) } -func unpackVarint(buf []byte) (int32, int) { +func packVarint(i int, buf []byte) int { + return vlq.PutInt(buf, int64(i)) +} + +func packVaruint(i uint, buf []byte) int { + return vlq.PutUint(buf, uint64(i)) +} + +func unpackVarint32(buf []byte) (int32, int) { i, ofs := vlq.Int(buf) return int32(i), ofs } +func unpackVarint(buf []byte) (int, int) { + i, ofs := vlq.Int(buf) + return int(i), ofs +} + +func unpackVaruint(buf []byte) (uint, int) { + i, ofs := vlq.Uint(buf) + return uint(i), ofs +} + const ( // number of bits in a big.Word wordBits = 32 << (uint64(^big.Word(0)) >> 63) @@ -1081,6 +1438,7 @@ const ( wordBytes = wordBits / 8 // max packed bigint words maxPackedBigintWords = (256 - wordBytes) / wordBytes + maxPackedBigintBytes = 249 ) // big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index f34ecefb..75608bcc 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -60,6 +60,17 @@ func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string return strconv.FormatInt(int64(len(h)), 16) + h } +func spentAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return strconv.FormatInt(int64(len(h)+1), 16) + h +} + +func bigintToHex(i *big.Int) string { + b := make([]byte, maxPackedBigintBytes) + l := packBigint(i, b) + return hex.EncodeToString(b[:l]) +} + // keyPair is used to compare given key value in DB with expected // for more complicated compares it is possible to specify CompareFunc type keyPair struct { @@ -105,7 +116,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { valOK = kp[i].CompareFunc(val) } if !valOK { - return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, cfNames[col], i, kp[i].Value) + return errors.Errorf("Incorrect value %v found in column %v row %v key %v, expecting %v", val, cfNames[col], i, key, kp[i].Value) } i++ } @@ -115,6 +126,38 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } +const ( + txidB1T1 = "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + txidB1T2 = "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + txidB2T1 = "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + txidB2T2 = "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + txidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + + addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac + addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac + addr3 = "mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw" // 76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac + addr4 = "2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS" // a9144a21db08fb6882cb152e1ff06780a430740f770487 + addr5 = "2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1" // a914e921fc4912a315078f370d959f2c4f7b6d2a683c87 + addr6 = "mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX" // 76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac + addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac + addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac + addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac +) + +var ( + satZero = big.NewInt(0) + satB1T1A1 = big.NewInt(100000000) + satB1T1A2 = big.NewInt(12345) + satB1T2A3 = big.NewInt(1234567890123) + satB1T2A4 = big.NewInt(1) + satB1T2A5 = big.NewInt(9876) + satB2T1A6 = big.NewInt(317283951061) + satB2T1A7 = big.NewInt(917283951061) + satB2T2A8 = big.NewInt(118641975500) + satB2T2A9 = big.NewInt(198641975500) + satB2T3A5 = big.NewInt(9000) +) + func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ @@ -123,44 +166,49 @@ func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d), + Hex: addressToPubKeyHex(addr1, t, d), }, + ValueSat: *satB1T1A1, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d), + Hex: addressToPubKeyHex(addr2, t, d), }, + ValueSat: *satB1T1A2, }, }, Blocktime: 22549300000, Time: 22549300000, }, bchain.Tx{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d), + Hex: addressToPubKeyHex(addr3, t, d), }, + ValueSat: *satB1T2A3, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d), + Hex: addressToPubKeyHex(addr4, t, d), }, + ValueSat: *satB1T2A4, }, bchain.Vout{ N: 2, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB1T2A5, }, }, Blocktime: 22549300001, @@ -178,14 +226,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vin: []bchain.Vin{ + // addr3 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 0, }, + // addr2 bchain.Vin{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: 1, }, }, @@ -193,30 +243,32 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d), + Hex: addressToPubKeyHex(addr6, t, d), }, + ValueSat: *satB2T1A6, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d), + Hex: addressToPubKeyHex(addr7, t, d), }, + ValueSat: *satB2T1A7, }, }, Blocktime: 22549400000, Time: 22549400000, }, bchain.Tx{ - Txid: "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", + Txid: txidB2T2, Vin: []bchain.Vin{ - // spending an output in the same block + // spending an output in the same block - addr6 bchain.Vin{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vout: 0, }, - // spending an output in the previous block + // spending an output in the previous block - addr4 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 1, }, }, @@ -224,14 +276,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d), + Hex: addressToPubKeyHex(addr8, t, d), }, + ValueSat: *satB2T2A8, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d), + Hex: addressToPubKeyHex(addr9, t, d), }, + ValueSat: *satB2T2A9, }, }, Blocktime: 22549400001, @@ -239,10 +293,11 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, // transaction from the same address in the previous block bchain.Tx{ - Txid: "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", + Txid: txidB2T3, Vin: []bchain.Vin{ + // addr5 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 2, }, }, @@ -250,8 +305,9 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB2T3A5, }, }, Blocktime: 22549400002, @@ -261,7 +317,7 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { } } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, }); err != nil { @@ -271,62 +327,51 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { } // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", - }) - }, + txidB1T1, + "00" + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), + nil, }, keyPair{ - "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "04", - }) - }, + txidB1T2, + "00" + "03" + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), + nil, }, }); err != nil { { t.Fatal(err) } } - // after disconnect there are no blockaddresses for the previous block - var blockAddressesKp []keyPair - if noBlockAddresses { - blockAddressesKp = []keyPair{} - } else { - // the values in cfBlockAddresses are in random order, must use CompareFunc - blockAddressesKp = []keyPair{ - keyPair{"000370d5", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "00", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", - }) - }, - }, + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A2), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A3), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A4), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A5), nil}, + }); err != nil { + { + t.Fatal(err) } } - if err := checkColumn(d, cfBlockAddresses, blockAddressesKp); err != nil { + if err := checkColumn(d, cfBlockTxids, []keyPair{ + keyPair{"000370d5", txidB1T1 + txidB1T2, nil}, + }); err != nil { { t.Fatal(err) } @@ -343,47 +388,66 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { } } if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, - keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil}, - keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil}, - keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil}, - keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d6", "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "00" + "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "01", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, + keyPair{addressToPubKeyHex(addr6, t, d) + "000370d6", txidB2T1 + "00" + txidB2T2 + "01", nil}, + keyPair{addressToPubKeyHex(addr7, t, d) + "000370d6", txidB2T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr8, t, d) + "000370d6", txidB2T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr9, t, d) + "000370d6", txidB2T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d6", txidB2T1 + "01", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d6", txidB2T1 + "03", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d6", txidB2T3 + "00" + txidB2T3 + "01", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d6", txidB2T2 + "03", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + txidB1T1, + "00" + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + spentAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), nil, }, keyPair{ - "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + txidB1T2, + "00" + "03" + + spentAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + spentAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + spentAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), nil, }, keyPair{ - "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", - }) - }, + txidB2T1, + "02" + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + + "02" + + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), + nil, }, keyPair{ - "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", + txidB2T2, + "02" + + addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + "02" + + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), + nil, + }, + keyPair{ + txidB2T3, + "01" + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + + "01" + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, }, }); err != nil { @@ -391,21 +455,23 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockAddresses, []keyPair{ - keyPair{"000370d6", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "00", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "00", - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02" + "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", - }) - }, - }, + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "02" + bigintToHex(satB1T1A2) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "02" + bigintToHex(satB1T2A3) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "02" + bigintToHex(satB1T2A4) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "02" + bigintToHex(satB1T2A5) + bigintToHex(satB2T3A5), nil}, + keyPair{addressToPubKeyHex(addr6, t, d), "02" + bigintToHex(satB2T1A6) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr7, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T1A7), nil}, + keyPair{addressToPubKeyHex(addr8, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A8), nil}, + keyPair{addressToPubKeyHex(addr9, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A9), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfBlockTxids, []keyPair{ + keyPair{"000370d6", txidB2T1 + txidB2T2 + txidB2T3, nil}, }); err != nil { { t.Fatal(err) @@ -487,12 +553,12 @@ func TestRocksDB_Index_UTXO(t *testing.T) { }) defer closeAndDestroyRocksDB(t, d) - // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column + // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column block1 := getTestUTXOBlock1(t, d) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, false) + verifyAfterUTXOBlock1(t, d) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block block2 := getTestUTXOBlock2(t, d) @@ -502,19 +568,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { verifyAfterUTXOBlock2(t, d) // get transactions for various addresses / low-high ranges - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225493, 225493, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, + verifyGetTransactions(t, d, addr2, 225493, 225493, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225494, 1000000, []txidVoutOutput{ - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 225494, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 500000, 1000000, []txidVoutOutput{}, nil) - verifyGetTransactions(t, d, "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", 0, true}, + verifyGetTransactions(t, d, addr2, 500000, 1000000, []txidVoutOutput{}, nil) + verifyGetTransactions(t, d, addr8, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T2, 0, true}, }, nil) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch")) @@ -569,7 +635,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, true) + verifyAfterUTXOBlock1(t, d) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { t.Fatal(err) @@ -661,13 +727,13 @@ func Test_unpackBlockAddresses(t *testing.T) { want2: [][]hexoutpoint{ []hexoutpoint{}, []hexoutpoint{ - hexoutpoint{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 0}, - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 3}, + hexoutpoint{txidB2T1, 0}, + hexoutpoint{txidB1T1, 3}, }, []hexoutpoint{}, []hexoutpoint{}, []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + hexoutpoint{txidB1T2, 1}, }, }, }, @@ -679,13 +745,13 @@ func Test_unpackBlockAddresses(t *testing.T) { []hexoutpoint{}, []hexoutpoint{}, []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 0}, + hexoutpoint{txidB1T2, 0}, }, []hexoutpoint{ - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1}, + hexoutpoint{txidB1T1, 1}, }, []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + hexoutpoint{txidB1T2, 1}, }, []hexoutpoint{}, []hexoutpoint{}, @@ -714,7 +780,7 @@ func Test_unpackBlockAddresses(t *testing.T) { for i, g := range got2 { ho := make([]hexoutpoint, len(g)) for j, o := range g { - ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.vout} + ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.index} } h2[i] = ho } @@ -741,12 +807,12 @@ func Test_packBigint_unpackBigint(t *testing.T) { { name: "0", bi: big.NewInt(0), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "1", bi: big.NewInt(1), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "54321", @@ -756,27 +822,27 @@ func Test_packBigint_unpackBigint(t *testing.T) { { name: "12345678", bi: big.NewInt(12345678), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "123456789123456789", bi: big.NewInt(123456789123456789), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "bigbig1", bi: bigbig1, - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "bigbig2", bi: bigbig2, - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "bigbigbig", bi: bigbigbig, - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), toobiglen: 242, }, } From fb93c9ff7a825ece512ac6ebcaac8247becd0970 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 Aug 2018 16:10:28 +0200 Subject: [PATCH 009/123] Add unit test for pack/unpack txAddress --- db/rocksdb.go | 59 ++++++++++-------- db/rocksdb_test.go | 148 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 178 insertions(+), 29 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 89c920e5..dc33debb 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -478,17 +478,7 @@ func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAd varBuf := make([]byte, maxPackedBigintBytes) buf := make([]byte, 1024) for txID, ta := range am { - buf = buf[:0] - l := packVaruint(uint(len(ta.inputs)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ta.inputs { - buf = appendTxAddress(buf, varBuf, &ta.inputs[i]) - } - l = packVaruint(uint(len(ta.outputs)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ta.outputs { - buf = appendTxAddress(buf, varBuf, &ta.outputs[i]) - } + buf = packTxAddresses(ta, buf, varBuf) wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) } return nil @@ -508,19 +498,6 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa return nil } -func appendTxAddress(buf []byte, varBuf []byte, txa *txAddress) []byte { - la := len(txa.addrID) - if txa.spent { - la = ^la - } - l := packVarint(la, varBuf) - buf = append(buf, varBuf[:l]...) - buf = append(buf, txa.addrID...) - l = packBigint(&txa.valueSat, varBuf) - buf = append(buf, varBuf[:l]...) - return buf -} - func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bchain.Block, txids [][]byte) error { pl := d.chainParser.PackedTxidLen() buf := make([]byte, pl*len(txids)) @@ -582,6 +559,38 @@ func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { if len(buf) < 2 { return nil, nil } + return unpackTxAddresses(buf) +} + +func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { + buf = buf[:0] + l := packVaruint(uint(len(ta.inputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.inputs { + buf = appendTxAddress(&ta.inputs[i], buf, varBuf) + } + l = packVaruint(uint(len(ta.outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.outputs { + buf = appendTxAddress(&ta.outputs[i], buf, varBuf) + } + return buf +} + +func appendTxAddress(txa *txAddress, buf []byte, varBuf []byte) []byte { + la := len(txa.addrID) + if txa.spent { + la = ^la + } + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txa.addrID...) + l = packBigint(&txa.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func unpackTxAddresses(buf []byte) (*txAddresses, error) { ta := txAddresses{} inputs, l := unpackVaruint(buf) ta.inputs = make([]txAddress, inputs) @@ -601,7 +610,7 @@ func unpackTxAddress(ta *txAddress, buf []byte) int { al, l := unpackVarint(buf) if al < 0 { ta.spent = true - al ^= al + al = ^al } ta.addrID = make([]byte, al) copy(ta.addrID, buf[l:l+al]) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 75608bcc..52aa3b29 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -23,6 +23,13 @@ import ( // for number n, the packing is: 2*n if n>=0 else 2*(-n)-1 // takes only 1 byte if abs(n)<127 +func bitcoinTestnetParser() *btc.BitcoinParser { + return &btc.BitcoinParser{ + BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, + Params: btc.GetChainParams("test"), + } +} + func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { tmp, err := ioutil.TempDir("", "testdb") if err != nil { @@ -546,10 +553,7 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ - BitcoinParser: &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, - Params: btc.GetChainParams("test"), - }, + BitcoinParser: bitcoinTestnetParser(), }) defer closeAndDestroyRocksDB(t, d) @@ -880,3 +884,139 @@ func Test_packBigint_unpackBigint(t *testing.T) { }) } } + +func addressToOutput(addr string, parser *btc.BitcoinParser) []byte { + b, err := parser.AddressToOutputScript(addr) + if err != nil { + panic(err) + } + return b +} + +func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { + parser := bitcoinTestnetParser() + tests := []struct { + name string + hex string + data *txAddresses + }{ + { + name: "1", + hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b7002c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + data: &txAddresses{ + inputs: []txAddress{ + { + addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + valueSat: *big.NewInt(0), + }, + { + addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), + valueSat: *big.NewInt(1234123421342341234), + }, + }, + outputs: []txAddress{ + { + addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + valueSat: *big.NewInt(1), + spent: true, + }, + }, + }, + }, + { + name: "2", + hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c02ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c02ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + data: &txAddresses{ + inputs: []txAddress{ + { + addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + valueSat: *big.NewInt(9011000000), + }, + { + addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + valueSat: *big.NewInt(8011000000), + }, + { + addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + valueSat: *big.NewInt(7011000000), + }, + }, + outputs: []txAddress{ + { + addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + valueSat: *big.NewInt(5011000000), + spent: true, + }, + { + addrID: addressToOutput("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + valueSat: *big.NewInt(6011000000), + }, + { + addrID: addressToOutput("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + valueSat: *big.NewInt(7011000000), + spent: true, + }, + { + addrID: addressToOutput("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + valueSat: *big.NewInt(999900000), + }, + { + addrID: addressToOutput("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + valueSat: *big.NewInt(5000000000), + spent: true, + }, + }, + }, + }, + { + name: "empty address", + hex: "01000204d2020002162e010162", + data: &txAddresses{ + inputs: []txAddress{ + { + addrID: []byte{}, + valueSat: *big.NewInt(1234), + }, + }, + outputs: []txAddress{ + { + addrID: []byte{}, + valueSat: *big.NewInt(5678), + }, + { + addrID: []byte{}, + valueSat: *big.NewInt(98), + spent: true, + }, + }, + }, + }, + { + name: "empty", + hex: "0000", + data: &txAddresses{ + inputs: []txAddress{}, + outputs: []txAddress{}, + }, + }, + } + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := packTxAddresses(tt.data, buf, varBuf) + hex := hex.EncodeToString(b) + if !reflect.DeepEqual(hex, tt.hex) { + t.Errorf("packTxAddresses() = %v, want %v", hex, tt.hex) + } + got1, err := unpackTxAddresses(b) + if err != nil { + t.Errorf("unpackTxAddresses() error = %v", err) + return + } + if !reflect.DeepEqual(got1, tt.data) { + t.Errorf("unpackTxAddresses() = %+v, want %+v", got1, tt.data) + } + }) + } +} From a04e38f0f7bab2d41223534bd4bdabe43f326f3a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 Aug 2018 17:46:23 +0200 Subject: [PATCH 010/123] Improve handling of invalid addresses --- db/rocksdb.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index dc33debb..e71f79d7 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -324,7 +324,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) } } else { - glog.Infof("rocksdb: block %d, skipping addrID of length %d", block.Height, len(addrID)) + glog.Infof("rocksdb: height %d, tx %v, vout %v, skipping addrID of length %d", block.Height, tx.Txid, i, len(addrID)) } continue } @@ -397,8 +397,12 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } tai.addrID = ot.addrID tai.valueSat = ot.valueSat - // mark the output tx as spent + // mark the output as spent in tx ot.spent = true + if len(ot.addrID) == 0 { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } strAddrID := string(ot.addrID) // check that the address was used already in this block o, processed := addresses[strAddrID] @@ -428,10 +432,11 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) if ab.balanceSat.Sign() < 0 { ad, err := d.chainParser.OutputScriptToAddresses(ot.addrID) + had := hex.EncodeToString(ot.addrID) if err != nil { - glog.Warningf("rocksdb: unparsable address reached negative balance %v, resetting to 0. Parser error %v", ab.balanceSat.String(), err) + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative balance %v, resetting to 0. Parser error %v", had, ab.balanceSat.String(), err) } else { - glog.Warningf("rocksdb: address %v reached negative balance %v, resetting to 0", ab.balanceSat.String(), ad) + glog.Warningf("rocksdb: address %v hex '%v' reached negative balance %v, resetting to 0", ad, had, ab.balanceSat.String()) } ab.balanceSat.SetInt64(0) } From 7e11a4e615ff947e7fcef5b5567eedec9ee44196 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 3 Aug 2018 15:04:49 +0200 Subject: [PATCH 011/123] Set LZ4HC compression for all columns --- db/rocksdb.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index e71f79d7..ebee699b 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -88,7 +88,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) optsLZ4.SetBytesPerSync(1 << 20) // 1MB optsLZ4.SetWriteBufferSize(1 << 27) // 128MB optsLZ4.SetMaxOpenFiles(25000) - optsLZ4.SetCompression(gorocksdb.LZ4Compression) + optsLZ4.SetCompression(gorocksdb.LZ4HCCompression) // opts for addresses are different: // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter @@ -105,10 +105,10 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) optsAddresses.SetBytesPerSync(1 << 20) // 1MB optsAddresses.SetWriteBufferSize(1 << 27) // 128MB optsAddresses.SetMaxOpenFiles(25000) - optsAddresses.SetCompression(gorocksdb.NoCompression) + optsAddresses.SetCompression(gorocksdb.LZ4HCCompression) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions - fcOptions := []*gorocksdb.Options{optsNoCompression, optsNoCompression, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} + fcOptions := []*gorocksdb.Options{optsLZ4, optsLZ4, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions) if err != nil { From 8e3c7f851bb7770de9aa9e7dbe5949b32d98c670 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 3 Aug 2018 19:26:16 +0200 Subject: [PATCH 012/123] Clean up the rocksdb sources and disconnect UTXO blocks WIP --- db/rocksdb.go | 517 ++++++++++++--------------------------------- db/rocksdb_test.go | 156 +------------- db/sync.go | 14 +- 3 files changed, 148 insertions(+), 539 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index ebee699b..7ce84b54 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -52,10 +52,6 @@ const ( cfAddressBalance cfBlockTxids cfTransactions - - // to be removed, kept temporarily so that the code is compilable - cfUnspentTxs - cfBlockAddresses ) var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"} @@ -257,7 +253,12 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return err } if isUTXO { - if err := d.writeAddressesUTXO(wb, block, op); err != nil { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } + if err := d.writeAddressesUTXO(wb, block); err != nil { return err } } else { @@ -293,12 +294,7 @@ type addrBalance struct { balanceSat big.Int } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } +func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { addresses := make(map[string][]outpoint) blockTxids := make([][]byte, len(block.Txs)) txAddressesMap := make(map[string]*txAddresses) @@ -443,21 +439,16 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } } - if op == opInsert { - if err := d.storeAddresses(wb, block, addresses); err != nil { - return err - } - if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { - return err - } - if err := d.storeBalances(wb, balances); err != nil { - return err - } - if err := d.storeAndCleanupBlockTxids(wb, block, blockTxids); err != nil { - return err - } + if err := d.storeAddresses(wb, block, addresses); err != nil { + return err } - return nil + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + return d.storeAndCleanupBlockTxids(wb, block, blockTxids) } func processedInTx(o []outpoint, btxID []byte) bool { @@ -493,12 +484,17 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa // allocate buffer big enough for number of txs + 2 bigints buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) for addrID, ab := range abm { - l := packVaruint(uint(ab.txs), buf) - ll := packBigint(&ab.sentSat, buf[l:]) - l += ll - ll = packBigint(&ab.balanceSat, buf[l:]) - l += ll - wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + // balance with 0 transactions is removed from db - happens in disconnect + if ab == nil || ab.txs <= 0 { + wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrID)) + } else { + l := packVaruint(uint(ab.txs), buf) + ll := packBigint(&ab.sentSat, buf[l:]) + l += ll + ll = packBigint(&ab.balanceSat, buf[l:]) + l += ll + wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + } } return nil } @@ -532,6 +528,22 @@ func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bch return nil } +func (d *RocksDB) getBlockTxids(height uint32) ([][]byte, error) { + pl := d.chainParser.PackedTxidLen() + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], packUint(height)) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + txids := make([][]byte, len(buf)/pl) + for i := 0; i < len(txids); i++ { + txid := make([]byte, pl) + copy(txid, buf[pl*i:]) + } + return txids, nil +} + func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) if err != nil { @@ -651,70 +663,9 @@ func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { return outpoints, nil } -//////////////////////// - -func (d *RocksDB) packBlockAddress(addrID []byte, spentTxs map[string][]outpoint) []byte { - vBuf := make([]byte, vlq.MaxLen32) - vl := packVarint(len(addrID), vBuf) - blockAddress := append([]byte(nil), vBuf[:vl]...) - blockAddress = append(blockAddress, addrID...) - if spentTxs == nil { - } else { - addrUnspentTxs := spentTxs[string(addrID)] - vl = packVarint(len(addrUnspentTxs), vBuf) - blockAddress = append(blockAddress, vBuf[:vl]...) - buf := d.packOutpoints(addrUnspentTxs) - blockAddress = append(blockAddress, buf...) - } - return blockAddress -} - -func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, addresses map[string][]outpoint, spentTxs map[string][]outpoint) error { - keep := d.chainParser.KeepBlockAddresses() - blockAddresses := make([]byte, 0) - for addrID, outpoints := range addresses { - baddrID := []byte(addrID) - key := packAddressKey(baddrID, block.Height) - switch op { - case opInsert: - val := d.packOutpoints(outpoints) - wb.PutCF(d.cfh[cfAddresses], key, val) - if keep > 0 { - // collect all addresses be stored in blockaddresses - // they are used in disconnect blocks - blockAddress := d.packBlockAddress(baddrID, spentTxs) - blockAddresses = append(blockAddresses, blockAddress...) - } - case opDelete: - wb.DeleteCF(d.cfh[cfAddresses], key) - } - } - if keep > 0 && op == opInsert { - // write new block address and txs spent in this block - key := packUint(block.Height) - wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses) - // cleanup old block address - if block.Height > uint32(keep) { - for rh := block.Height - uint32(keep); rh < block.Height; rh-- { - key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return err - } - if val.Size() == 0 { - break - } - val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key) - } - } - } - return nil -} - func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { if len(addrID) > 0 { - if len(addrID) > 1024 { + if len(addrID) > maxAddrIDLen { glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) } else { strAddrID := string(addrID) @@ -731,153 +682,6 @@ func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records m return nil } -func (d *RocksDB) getUnspentTx(btxID []byte) ([]byte, error) { - // find it in db, in the column cfUnspentTxs - val, err := d.db.GetCF(d.ro, d.cfh[cfUnspentTxs], btxID) - if err != nil { - return nil, err - } - defer val.Free() - data := append([]byte(nil), val.Data()...) - return data, nil -} - -func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) []byte { - // resize the addr buffer if necessary by a new estimate - if cap(txAddrs)-len(txAddrs) < 2*vlq.MaxLen32+len(addrID) { - txAddrs = append(txAddrs, make([]byte, vlq.MaxLen32+len(addrID)+remaining*32)...)[:len(txAddrs)] - } - // addrID is packed as number of bytes of the addrID + bytes of addrID + vout - lv := packVarint(len(addrID), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - txAddrs = append(txAddrs, addrID...) - lv = packVarint(int(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - return txAddrs -} - -func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, []byte) { - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(unspentAddrs); { - l, lv1 := unpackVarint(unspentAddrs[i:]) - // index of vout of address in unspentAddrs - j := i + int(l) + lv1 - if j >= len(unspentAddrs) { - glog.Error("rocksdb: Inconsistent data in unspentAddrs ", hex.EncodeToString(unspentAddrs), ", ", vout) - return nil, unspentAddrs - } - n, lv2 := unpackVarint(unspentAddrs[j:]) - if uint32(n) == vout { - addrID := append([]byte(nil), unspentAddrs[i+lv1:j]...) - unspentAddrs = append(unspentAddrs[:i], unspentAddrs[j+lv2:]...) - return addrID, unspentAddrs - } - i = j + lv2 - } - return nil, unspentAddrs -} - -func (d *RocksDB) writeAddressesUTXO_old(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } - addresses := make(map[string][]outpoint) - unspentTxs := make(map[string][]byte) - thisBlockTxs := make(map[string]struct{}) - btxIDs := make([][]byte, len(block.Txs)) - // first process all outputs, build mapping of addresses to outpoints and mappings of unspent txs to addresses - for txi, tx := range block.Txs { - btxID, err := d.chainParser.PackTxid(tx.Txid) - if err != nil { - return err - } - btxIDs[txi] = btxID - // preallocate estimated size of addresses (32 bytes is 1 byte length of addrID, 25 bytes addrID, 1-2 bytes vout and reserve) - txAddrs := make([]byte, 0, len(tx.Vout)*32) - for i, output := range tx.Vout { - addrID, err := d.chainParser.GetAddrIDFromVout(&output) - if err != nil { - // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) - if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) - } - continue - } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(output.N), block.Height) - if err != nil { - return err - } - txAddrs = appendPackedAddrID(txAddrs, addrID, output.N, len(tx.Vout)-i) - } - stxID := string(btxID) - unspentTxs[stxID] = txAddrs - thisBlockTxs[stxID] = struct{}{} - } - // locate addresses spent by this tx and remove them from unspent addresses - // keep them so that they be stored for DisconnectBlock functionality - spentTxs := make(map[string][]outpoint) - for txi, tx := range block.Txs { - spendingTxid := btxIDs[txi] - for i, input := range tx.Vin { - btxID, err := d.chainParser.PackTxid(input.Txid) - if err != nil { - // do not process inputs without input txid - if err == bchain.ErrTxidMissing { - continue - } - return err - } - // find the tx in current block or already processed - stxID := string(btxID) - unspentAddrs, exists := unspentTxs[stxID] - if !exists { - // else find it in previous blocks - unspentAddrs, err = d.getUnspentTx(btxID) - if err != nil { - return err - } - if unspentAddrs == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v missing in unspentTxs", block.Height, tx.Txid, input.Txid, input.Vout, i) - continue - } - } - var addrID []byte - addrID, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout) - if addrID == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v not found in unspentAddrs", block.Height, tx.Txid, input.Txid, input.Vout, i) - continue - } - // record what was spent in this tx - // skip transactions that were created in this block - if _, exists := thisBlockTxs[stxID]; !exists { - saddrID := string(addrID) - rut := spentTxs[saddrID] - rut = append(rut, outpoint{btxID, int32(input.Vout)}) - spentTxs[saddrID] = rut - } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, spendingTxid, int32(^i), block.Height) - if err != nil { - return err - } - unspentTxs[stxID] = unspentAddrs - } - } - if err := d.writeAddressRecords(wb, block, op, addresses, spentTxs); err != nil { - return err - } - // save unspent txs from current block - for tx, val := range unspentTxs { - if len(val) == 0 { - wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) - } else { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) - } - } - return nil -} - func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { addresses := make(map[string][]outpoint) for _, tx := range block.Txs { @@ -914,75 +718,19 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. } } } - return d.writeAddressRecords(wb, block, op, addresses, nil) -} - -func (d *RocksDB) unpackBlockAddresses(buf []byte) ([][]byte, [][]outpoint, error) { - addresses := make([][]byte, 0) - outpointsArray := make([][]outpoint, 0) - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(buf); { - l, lv := unpackVarint(buf[i:]) - j := i + int(l) + lv - if j > len(buf) { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addrID := append([]byte(nil), buf[i+lv:j]...) - outpoints, ol, err := d.unpackNOutpoints(buf[j:]) - if err != nil { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addresses = append(addresses, addrID) - outpointsArray = append(outpointsArray, outpoints) - i = j + ol - } - return addresses, outpointsArray, nil -} - -func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - n, p := unpackVarint32(buf) - outpoints := make([]outpoint, n) - for i := int32(0); i < n; i++ { - if p+txidUnpackedLen >= len(buf) { - return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") - } - btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) - p += txidUnpackedLen - vout, voutLen := unpackVarint32(buf[p:]) - p += voutLen - outpoints[i] = outpoint{ - btxID: btxID, - index: vout, + for addrID, outpoints := range addresses { + key := packAddressKey([]byte(addrID), block.Height) + switch op { + case opInsert: + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + case opDelete: + wb.DeleteCF(d.cfh[cfAddresses], key) } } - return outpoints, p, nil + return nil } -func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { - btxid, err := d.chainParser.PackTxid(txid) - if err != nil { - return nil, err - } - bv := make([]byte, vlq.MaxLen32) - l := packVarint32(vout, bv) - buf := make([]byte, 0, l+len(btxid)) - buf = append(buf, btxid...) - buf = append(buf, bv[:l]...) - return buf, nil -} - -func (d *RocksDB) unpackOutpoint(buf []byte) (string, int32, int) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - txid, _ := d.chainParser.UnpackTxid(buf[:txidUnpackedLen]) - vout, o := unpackVarint32(buf[txidUnpackedLen:]) - return txid, vout, txidUnpackedLen + o -} - -////////////////////////////////// - // Block index // GetBestBlock returns the block hash of the block with highest height in the db @@ -1032,18 +780,7 @@ func (d *RocksDB) writeHeight( return nil } -func (d *RocksDB) getBlockAddresses(key []byte) ([][]byte, [][]outpoint, error) { - b, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return nil, nil, err - } - defer b.Free() - // block is missing in DB - if b.Data() == nil { - return nil, nil, errors.New("Block addresses missing") - } - return d.unpackBlockAddresses(b.Data()) -} +// Disconnect blocks func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]byte, error) { glog.Infof("db: doing full scan of addresses column") @@ -1090,93 +827,107 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -// DisconnectBlockRange removes all data belonging to blocks in range lower-higher -// it finds the data in blockaddresses column if available, -// otherwise by doing quite slow full scan of addresses column -func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys := [][]byte{} - addrOutpoints := [][]byte{} - addrUnspentOutpoints := [][]outpoint{} - keep := d.chainParser.KeepBlockAddresses() - var err error - if keep > 0 { - for height := lower; height <= higher; height++ { - addresses, unspentOutpoints, err := d.getBlockAddresses(packUint(height)) +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, txAddressesToUpdate map[string]*txAddresses, txsToDelete map[string]struct{}, balances map[string]*addrBalance) error { + findBalance := func(addrID []byte) (*addrBalance, error) { + var err error + s := string(addrID) + b, found := balances[s] + if !found { + b, err = d.getAddressBalance(addrID) if err != nil { - glog.Error(err) - return err - } - for i, addrID := range addresses { - addrKey := packAddressKey(addrID, height) - val, err := d.db.GetCF(d.ro, d.cfh[cfAddresses], addrKey) - if err != nil { - glog.Error(err) - return err - } - addrKeys = append(addrKeys, addrKey) - av := append([]byte(nil), val.Data()...) - val.Free() - addrOutpoints = append(addrOutpoints, av) - addrUnspentOutpoints = append(addrUnspentOutpoints, unspentOutpoints[i]) + return nil, err } + balances[s] = b } - } else { - addrKeys, addrOutpoints, err = d.allAddressesScan(lower, higher) + return b, nil + } + for _, t := range txa.inputs { + b, err := findBalance(t.addrID) if err != nil { return err } + if b != nil { + + } } + return nil +} + +// DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher +// if they are in the range kept in the cfBlockTxs column +func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + blocksTxids := make([][][]byte, higher-lower+1) + for height := lower; height <= higher; height++ { + blockTxids, err := d.getBlockTxids(height) + if err != nil { + return err + } + if len(blockTxids) == 0 { + return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) + } + blocksTxids[height-lower] = blockTxids + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + txAddressesToUpdate := make(map[string]*txAddresses) + txsToDelete := make(map[string]struct{}) + balances := make(map[string]*addrBalance) + for height := higher; height >= lower; height-- { + blockTxids := blocksTxids[height-lower] + for _, txid := range blockTxids { + txa, err := d.getTxAddresses(txid) + if err != nil { + return err + } + s := string(txid) + txsToDelete[s] = struct{}{} + if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, txsToDelete, balances); err != nil { + return err + } + } + key := packUint(height) + wb.DeleteCF(d.cfh[cfBlockTxids], key) + wb.DeleteCF(d.cfh[cfHeight], key) + } + d.storeTxAddresses(wb, txAddressesToUpdate) + d.storeBalances(wb, balances) + for s := range txsToDelete { + b := []byte(s) + wb.DeleteCF(d.cfh[cfTransactions], b) + wb.DeleteCF(d.cfh[cfTxAddresses], b) + } + err := d.db.Write(d.wo, wb) + if err == nil { + glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) + } + return err +} + +// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks +// it is very slow operation +func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + addrKeys, _, err := d.allAddressesScan(lower, higher) + if err != nil { + return err + } glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - unspentTxs := make(map[string][]byte) - for addrIndex, addrKey := range addrKeys { + for _, addrKey := range addrKeys { if glog.V(2) { glog.Info("address ", hex.EncodeToString(addrKey)) } // delete address:height from the index wb.DeleteCF(d.cfh[cfAddresses], addrKey) - addrID, _, err := unpackAddressKey(addrKey) - if err != nil { - return err - } - // recreate unspentTxs, which were spent by this block (that is being disconnected) - for _, o := range addrUnspentOutpoints[addrIndex] { - stxID := string(o.btxID) - txAddrs, exists := unspentTxs[stxID] - if !exists { - txAddrs, err = d.getUnspentTx(o.btxID) - if err != nil { - return err - } - } - txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.index), 1) - unspentTxs[stxID] = txAddrs - } - // delete unspentTxs from this block - outpoints, err := d.unpackOutpoints(addrOutpoints[addrIndex]) - if err != nil { - return err - } - for _, o := range outpoints { - wb.DeleteCF(d.cfh[cfUnspentTxs], o.btxID) - d.internalDeleteTx(wb, o.btxID) - } - } - for key, val := range unspentTxs { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(key), val) } for height := lower; height <= higher; height++ { if glog.V(2) { glog.Info("height ", height) } - key := packUint(height) - if keep > 0 { - wb.DeleteCF(d.cfh[cfBlockAddresses], key) - } - wb.DeleteCF(d.cfh[cfHeight], key) + wb.DeleteCF(d.cfh[cfHeight], packUint(height)) } err = d.db.Write(d.wo, wb) if err == nil { diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 52aa3b29..4a96e769 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -632,9 +632,16 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } verifyAfterUTXOBlock2(t, d) + // try to disconnect both blocks, however only the last one is kept, it is not possible + err = d.DisconnectBlockRangeUTXO(225493, 225494) + if err == nil || err.Error() != "Cannot disconnect blocks with height 225493 and lower. It is necessary to rebuild index." { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs // and that the cached tx is removed - err = d.DisconnectBlockRange(225494, 225494) + err = d.DisconnectBlockRangeUTXO(225494, 225494) if err != nil { t.Fatal(err) } @@ -648,153 +655,6 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } -func Test_findAndRemoveUnspentAddr(t *testing.T) { - type args struct { - unspentAddrs string - vout uint32 - } - tests := []struct { - name string - args args - want string - want2 string - }{ - { - name: "3", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 3, - }, - want: "64635167006868", - want2: "029c0010517a0115887452870212709393588893935687040e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - }, - { - name: "10", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 10, - }, - want: "61", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112", - }, - { - name: "not there", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 11, - }, - want: "", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.unspentAddrs) - if err != nil { - panic(err) - } - got, got2 := findAndRemoveUnspentAddr(b, tt.args.vout) - h := hex.EncodeToString(got) - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("findAndRemoveUnspentAddr() got = %v, want %v", h, tt.want) - } - h2 := hex.EncodeToString(got2) - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("findAndRemoveUnspentAddr() got2 = %v, want %v", h2, tt.want2) - } - }) - } -} - -type hexoutpoint struct { - txID string - vout int32 -} - -func Test_unpackBlockAddresses(t *testing.T) { - d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}}) - defer closeAndDestroyRocksDB(t, d) - type args struct { - buf string - } - tests := []struct { - name string - args args - want []string - want2 [][]hexoutpoint - wantErr bool - }{ - { - name: "1", - args: args{"029c0010517a011588745287047c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d250000b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa38400612709393588893935687000e64635167006868000e7651935188008702effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac7502"}, - want: []string{"9c", "517a011588745287", "709393588893935687", "64635167006868", "76519351880087"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{txidB2T1, 0}, - hexoutpoint{txidB1T1, 3}, - }, - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{txidB1T2, 1}, - }, - }, - }, - { - name: "1", - args: args{"3276A914B434EB0C1A3B7A02E8A29CC616E791EF1E0BF51F88AC003276A9143F8BA3FDA3BA7B69F5818086E12223C6DD25E3C888AC003276A914A08EAE93007F22668AB5E4A9C83C8CD1C325E3E088AC02EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75003276A9148BDF0AA3C567AA5975C2E61321B8BEBBE7293DF688AC0200B2C06055E5E90E9C82BD4181FDE310104391A7FA4F289B1704E5D90CAA3840022EA9144A21DB08FB6882CB152E1FF06780A430740F77048702EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75023276A914CCAAAF374E1B06CB83118453D102587B4273D09588AC003276A9148D802C045445DF49613F6A70DDD2E48526F3701F88AC00"}, - want: []string{"76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac", "76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac", "76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac", "76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac", "a9144a21db08fb6882cb152e1ff06780a430740f770487", "76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac", "76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{txidB1T2, 0}, - }, - []hexoutpoint{ - hexoutpoint{txidB1T1, 1}, - }, - []hexoutpoint{ - hexoutpoint{txidB1T2, 1}, - }, - []hexoutpoint{}, - []hexoutpoint{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.buf) - if err != nil { - panic(err) - } - got, got2, err := d.unpackBlockAddresses(b) - if (err != nil) != tt.wantErr { - t.Errorf("unpackBlockAddresses() error = %v, wantErr %v", err, tt.wantErr) - return - } - h := make([]string, len(got)) - for i, g := range got { - h[i] = hex.EncodeToString(g) - } - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h, tt.want) - } - h2 := make([][]hexoutpoint, len(got2)) - for i, g := range got2 { - ho := make([]hexoutpoint, len(g)) - for j, o := range g { - ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.index} - } - h2[i] = ho - } - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h2, tt.want2) - } - }) - } -} - func Test_packBigint_unpackBigint(t *testing.T) { bigbig1, _ := big.NewInt(0).SetString("123456789123456789012345", 10) bigbig2, _ := big.NewInt(0).SetString("12345678912345678901234512389012345123456789123456789012345123456789123456789012345", 10) diff --git a/db/sync.go b/db/sync.go index a664e94f..fecfa09c 100644 --- a/db/sync.go +++ b/db/sync.go @@ -346,25 +346,23 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { } // DisconnectBlocks removes all data belonging to blocks in range lower-higher, -// using block data from blockchain, if they are available, -// otherwise doing full scan func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) - // if the chain uses Block to Addresses mapping, always use DisconnectBlockRange - if w.chain.GetChainParser().KeepBlockAddresses() > 0 { - return w.db.DisconnectBlockRange(lower, higher) + // if the chain is UTXO, always use DisconnectBlockRange + if w.chain.GetChainParser().IsUTXOChain() { + return w.db.DisconnectBlockRangeUTXO(lower, higher) } blocks := make([]*bchain.Block, len(hashes)) var err error - // get all blocks first to see if we can avoid full scan + // try to get all blocks first to see if we can avoid full scan for i, hash := range hashes { blocks[i], err = w.chain.GetBlock(hash, 0) if err != nil { // cannot get a block, we must do full range scan - return w.db.DisconnectBlockRange(lower, higher) + return w.db.DisconnectBlockRangeNonUTXO(lower, higher) } } - // then disconnect one after another + // got all blocks to be disconnected, disconnect them one after another for i, block := range blocks { glog.Info("Disconnecting block ", (int(higher) - i), " ", block.Hash) if err = w.db.DisconnectBlock(block); err != nil { From 78d2c085a97805ebefb8b08b974dbeed057eda26 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 15 Aug 2018 16:07:09 +0200 Subject: [PATCH 013/123] Log expected db data version --- db/rocksdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 7ce84b54..aa53d74c 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -116,7 +116,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { - glog.Infof("rocksdb: open %s", path) + glog.Infof("rocksdb: open %s, version %v", path, dbVersion) db, cfh, err := openDB(path) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() From f5b29b3a17e12c98e1f675e6f0f088b441978a05 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 15 Aug 2018 19:22:26 +0200 Subject: [PATCH 014/123] Implement index v3 WIP --- db/rocksdb.go | 187 ++++++++++++++++++++++++++++++++++----------- db/rocksdb_test.go | 40 +++++----- 2 files changed, 167 insertions(+), 60 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index aa53d74c..dfff330a 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -277,15 +277,21 @@ type outpoint struct { index int32 } -type txAddress struct { +type txInput struct { + addrID []byte + vout uint32 + valueSat big.Int +} + +type txOutput struct { addrID []byte spent bool valueSat big.Int } type txAddresses struct { - inputs []txAddress - outputs []txAddress + inputs []txInput + outputs []txOutput } type addrBalance struct { @@ -294,6 +300,17 @@ type addrBalance struct { balanceSat big.Int } +func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText string) { + ad, err := d.chainParser.OutputScriptToAddresses(addrID) + had := hex.EncodeToString(addrID) + if err != nil { + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", had, logText, valueSat.String(), err) + } else { + glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, had, logText, valueSat.String()) + } + valueSat.SetInt64(0) +} + func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { addresses := make(map[string][]outpoint) blockTxids := make([][]byte, len(block.Txs)) @@ -307,7 +324,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } blockTxids[txi] = btxID ta := txAddresses{} - ta.outputs = make([]txAddress, len(tx.Vout)) + ta.outputs = make([]txOutput, len(tx.Vout)) txAddressesMap[string(btxID)] = &ta for i, output := range tx.Vout { tao := &ta.outputs[i] @@ -358,7 +375,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo for txi, tx := range block.Txs { spendingTxid := blockTxids[txi] ta := txAddressesMap[string(spendingTxid)] - ta.inputs = make([]txAddress, len(tx.Vin)) + ta.inputs = make([]txInput, len(tx.Vin)) for i, input := range tx.Vin { tai := &ta.inputs[i] btxID, err := d.chainParser.PackTxid(input.Txid) @@ -392,6 +409,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo continue } tai.addrID = ot.addrID + tai.vout = input.Vout tai.valueSat = ot.valueSat // mark the output as spent in tx ot.spent = true @@ -427,14 +445,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) if ab.balanceSat.Sign() < 0 { - ad, err := d.chainParser.OutputScriptToAddresses(ot.addrID) - had := hex.EncodeToString(ot.addrID) - if err != nil { - glog.Warningf("rocksdb: unparsable address hex '%v' reached negative balance %v, resetting to 0. Parser error %v", had, ab.balanceSat.String(), err) - } else { - glog.Warningf("rocksdb: address %v hex '%v' reached negative balance %v, resetting to 0", ad, had, ab.balanceSat.String()) - } - ab.balanceSat.SetInt64(0) + d.resetValueSatToZero(&ab.balanceSat, ot.addrID, "balance") } ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } @@ -584,25 +595,37 @@ func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { l := packVaruint(uint(len(ta.inputs)), varBuf) buf = append(buf, varBuf[:l]...) for i := range ta.inputs { - buf = appendTxAddress(&ta.inputs[i], buf, varBuf) + buf = appendTxInput(&ta.inputs[i], buf, varBuf) } l = packVaruint(uint(len(ta.outputs)), varBuf) buf = append(buf, varBuf[:l]...) for i := range ta.outputs { - buf = appendTxAddress(&ta.outputs[i], buf, varBuf) + buf = appendTxOutput(&ta.outputs[i], buf, varBuf) } return buf } -func appendTxAddress(txa *txAddress, buf []byte, varBuf []byte) []byte { - la := len(txa.addrID) - if txa.spent { +func appendTxInput(txi *txInput, buf []byte, varBuf []byte) []byte { + la := len(txi.addrID) + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txi.addrID...) + l = packBigint(&txi.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + l = packVaruint(uint(txi.vout), varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func appendTxOutput(txo *txOutput, buf []byte, varBuf []byte) []byte { + la := len(txo.addrID) + if txo.spent { la = ^la } l := packVarint(la, varBuf) buf = append(buf, varBuf[:l]...) - buf = append(buf, txa.addrID...) - l = packBigint(&txa.valueSat, varBuf) + buf = append(buf, txo.addrID...) + l = packBigint(&txo.valueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf } @@ -610,29 +633,41 @@ func appendTxAddress(txa *txAddress, buf []byte, varBuf []byte) []byte { func unpackTxAddresses(buf []byte) (*txAddresses, error) { ta := txAddresses{} inputs, l := unpackVaruint(buf) - ta.inputs = make([]txAddress, inputs) + ta.inputs = make([]txInput, inputs) for i := uint(0); i < inputs; i++ { - l += unpackTxAddress(&ta.inputs[i], buf[l:]) + l += unpackTxInput(&ta.inputs[i], buf[l:]) } outputs, ll := unpackVaruint(buf[l:]) l += ll - ta.outputs = make([]txAddress, outputs) + ta.outputs = make([]txOutput, outputs) for i := uint(0); i < outputs; i++ { - l += unpackTxAddress(&ta.outputs[i], buf[l:]) + l += unpackTxOutput(&ta.outputs[i], buf[l:]) } return &ta, nil } -func unpackTxAddress(ta *txAddress, buf []byte) int { +func unpackTxInput(ti *txInput, buf []byte) int { + al, l := unpackVarint(buf) + ti.addrID = make([]byte, al) + copy(ti.addrID, buf[l:l+al]) + al += l + ti.valueSat, l = unpackBigint(buf[al:]) + al += l + v, l := unpackVaruint(buf[al:]) + ti.vout = uint32(v) + return l + al +} + +func unpackTxOutput(to *txOutput, buf []byte) int { al, l := unpackVarint(buf) if al < 0 { - ta.spent = true + to.spent = true al = ^al } - ta.addrID = make([]byte, al) - copy(ta.addrID, buf[l:l+al]) + to.addrID = make([]byte, al) + copy(to.addrID, buf[l:l+al]) al += l - ta.valueSat, l = unpackBigint(buf[al:]) + to.valueSat, l = unpackBigint(buf[al:]) return l + al } @@ -759,11 +794,7 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { return d.chainParser.UnpackBlockHash(val.Data()) } -func (d *RocksDB) writeHeight( - wb *gorocksdb.WriteBatch, - block *bchain.Block, - op int, -) error { +func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { key := packUint(block.Height) switch op { @@ -776,7 +807,6 @@ func (d *RocksDB) writeHeight( case opDelete: wb.DeleteCF(d.cfh[cfHeight], key) } - return nil } @@ -827,12 +857,46 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, txAddressesToUpdate map[string]*txAddresses, txsToDelete map[string]struct{}, balances map[string]*addrBalance) error { - findBalance := func(addrID []byte) (*addrBalance, error) { +// get all transactions of the given address and match it to input to find spent output +func (d *RocksDB) findSpentTx(ti *txInput) ([]byte, *txAddresses, error) { + start := packAddressKey(ti.addrID, 0) + stop := packAddressKey(ti.addrID, ^uint32(0)) + it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) + defer it.Close() + for it.Seek(start); it.Valid(); it.Next() { + key := it.Key().Data() + val := it.Value().Data() + if bytes.Compare(key, stop) > 0 { + break + } + outpoints, err := d.unpackOutpoints(val) + if err != nil { + return nil, nil, err + } + for _, o := range outpoints { + // process only outputs that match + if o.index >= 0 && uint32(o.index) == ti.vout { + a, err := d.getTxAddresses(o.btxID) + if err != nil { + return nil, nil, err + } + if bytes.Equal(a.outputs[o.index].addrID, ti.addrID) { + return o.btxID, a, nil + } + } + } + } + return nil, nil, nil +} + +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, + txAddressesToUpdate map[string]*txAddresses, balances map[string]*addrBalance) error { + addresses := make(map[string]struct{}) + getAddressBalance := func(addrID []byte) (*addrBalance, error) { var err error s := string(addrID) - b, found := balances[s] - if !found { + b, fb := balances[s] + if !fb { b, err = d.getAddressBalance(addrID) if err != nil { return nil, err @@ -842,15 +906,52 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, return b, nil } for _, t := range txa.inputs { - b, err := findBalance(t.addrID) + s := string(t.addrID) + _, fa := addresses[s] + if !fa { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.addrID) if err != nil { return err } if b != nil { - + // subtract number of txs only once + if !fa { + b.txs-- + } + b.sentSat.Sub(&b.sentSat, &t.valueSat) + if b.sentSat.Sign() < 0 { + d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + } + b.balanceSat.Add(&b.balanceSat, &t.valueSat) } } - + for _, t := range txa.outputs { + s := string(t.addrID) + _, fa := addresses[s] + if !fa { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.addrID) + if err != nil { + return err + } + if b != nil { + // subtract number of txs only once + if !fa { + b.txs-- + } + b.balanceSat.Sub(&b.balanceSat, &t.valueSat) + if b.balanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + } + } + } + for a := range addresses { + key := packAddressKey([]byte(a), height) + wb.DeleteCF(d.cfh[cfAddresses], key) + } return nil } @@ -883,7 +984,7 @@ func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { } s := string(txid) txsToDelete[s] = struct{}{} - if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, txsToDelete, balances); err != nil { + if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, balances); err != nil { return err } } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 4a96e769..fe841e82 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -432,8 +432,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T1, "02" + - addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + - addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + "00" + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "01" + "02" + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), @@ -442,8 +442,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T2, "02" + - addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + - addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + "00" + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "01" + "02" + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), @@ -452,7 +452,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T3, "01" + - addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "02" + "01" + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, @@ -548,7 +548,7 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality // 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using blockaddresses column +// 6) Disconnect the block 2 using BlockTxids column // 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { @@ -762,19 +762,21 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }{ { name: "1", - hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b7002c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b700e0392c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a19987283bb55012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", data: &txAddresses{ - inputs: []txAddress{ + inputs: []txInput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), valueSat: *big.NewInt(0), + vout: 12345, }, { addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), valueSat: *big.NewInt(1234123421342341234), + vout: 56789, }, }, - outputs: []txAddress{ + outputs: []txOutput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), valueSat: *big.NewInt(1), @@ -785,23 +787,26 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "2", - hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c02ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c02ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c0012ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c0022ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec003052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", data: &txAddresses{ - inputs: []txAddress{ + inputs: []txInput{ { addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), valueSat: *big.NewInt(9011000000), + vout: 1, }, { addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), valueSat: *big.NewInt(8011000000), + vout: 2, }, { addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), valueSat: *big.NewInt(7011000000), + vout: 3, }, }, - outputs: []txAddress{ + outputs: []txOutput{ { addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), valueSat: *big.NewInt(5011000000), @@ -830,15 +835,16 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "empty address", - hex: "01000204d2020002162e010162", + hex: "01000204d201020002162e010162", data: &txAddresses{ - inputs: []txAddress{ + inputs: []txInput{ { addrID: []byte{}, valueSat: *big.NewInt(1234), + vout: 1, }, }, - outputs: []txAddress{ + outputs: []txOutput{ { addrID: []byte{}, valueSat: *big.NewInt(5678), @@ -855,8 +861,8 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { name: "empty", hex: "0000", data: &txAddresses{ - inputs: []txAddress{}, - outputs: []txAddress{}, + inputs: []txInput{}, + outputs: []txOutput{}, }, }, } From 745d7af92d65718370233884a55f179d2173509c Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 16 Aug 2018 15:31:11 +0200 Subject: [PATCH 015/123] Implement index v3 WIP --- db/rocksdb.go | 280 ++++++++++++++++++++++++++------------------- db/rocksdb_test.go | 79 ++++++++----- 2 files changed, 215 insertions(+), 144 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index dfff330a..08d1eb78 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -50,11 +50,11 @@ const ( cfAddresses cfTxAddresses cfAddressBalance - cfBlockTxids + cfBlockTxs cfTransactions ) -var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"} +var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -279,7 +279,6 @@ type outpoint struct { type txInput struct { addrID []byte - vout uint32 valueSat big.Int } @@ -300,6 +299,11 @@ type addrBalance struct { balanceSat big.Int } +type blockTxs struct { + btxID []byte + inputs []outpoint +} + func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText string) { ad, err := d.chainParser.OutputScriptToAddresses(addrID) had := hex.EncodeToString(addrID) @@ -313,16 +317,17 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { addresses := make(map[string][]outpoint) - blockTxids := make([][]byte, len(block.Txs)) + blockTxIDs := make([][]byte, len(block.Txs)) txAddressesMap := make(map[string]*txAddresses) balances := make(map[string]*addrBalance) // first process all outputs so that inputs can point to txs in this block - for txi, tx := range block.Txs { + for txi := range block.Txs { + tx := &block.Txs[txi] btxID, err := d.chainParser.PackTxid(tx.Txid) if err != nil { return err } - blockTxids[txi] = btxID + blockTxIDs[txi] = btxID ta := txAddresses{} ta.outputs = make([]txOutput, len(tx.Vout)) txAddressesMap[string(btxID)] = &ta @@ -372,8 +377,9 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } } // process inputs - for txi, tx := range block.Txs { - spendingTxid := blockTxids[txi] + for txi := range block.Txs { + tx := &block.Txs[txi] + spendingTxid := blockTxIDs[txi] ta := txAddressesMap[string(spendingTxid)] ta.inputs = make([]txInput, len(tx.Vin)) for i, input := range tx.Vin { @@ -406,10 +412,8 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ot := &ita.outputs[int(input.Vout)] if ot.spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) - continue } tai.addrID = ot.addrID - tai.vout = input.Vout tai.valueSat = ot.valueSat // mark the output as spent in tx ot.spent = true @@ -459,7 +463,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo if err := d.storeBalances(wb, balances); err != nil { return err } - return d.storeAndCleanupBlockTxids(wb, block, blockTxids) + return d.storeAndCleanupBlockTxs(wb, block) } func processedInTx(o []outpoint, btxID []byte) bool { @@ -510,22 +514,38 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa return nil } -func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bchain.Block, txids [][]byte) error { - pl := d.chainParser.PackedTxidLen() - buf := make([]byte, pl*len(txids)) - i := 0 - for _, txid := range txids { - copy(buf[i:], txid) - i += pl +func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { + buf := make([]byte, 0, d.chainParser.PackedTxidLen()*len(block.Txs)) + varBuf := make([]byte, vlq.MaxLen64) + for i := range block.Txs { + tx := &block.Txs[i] + o := make([]outpoint, len(tx.Vin)) + for v := range tx.Vin { + vin := &tx.Vin[v] + btxID, err := d.chainParser.PackTxid(vin.Txid) + if err != nil { + return err + } + o[v].btxID = btxID + o[v].index = int32(vin.Vout) + } + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return err + } + buf = append(buf, btxID...) + l := packVaruint(uint(len(o)), varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, d.packOutpoints(o)...) } key := packUint(block.Height) - wb.PutCF(d.cfh[cfBlockTxids], key, buf) + wb.PutCF(d.cfh[cfBlockTxs], key, buf) keep := d.chainParser.KeepBlockAddresses() // cleanup old block address if block.Height > uint32(keep) { for rh := block.Height - uint32(keep); rh < block.Height; rh-- { key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], key) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) if err != nil { return err } @@ -533,26 +553,41 @@ func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bch break } val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockTxids], key) + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) } } return nil } -func (d *RocksDB) getBlockTxids(height uint32) ([][]byte, error) { +func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { pl := d.chainParser.PackedTxidLen() - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], packUint(height)) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) if err != nil { return nil, err } defer val.Free() buf := val.Data() - txids := make([][]byte, len(buf)/pl) - for i := 0; i < len(txids); i++ { + bt := make([]blockTxs, 0) + for i := 0; i < len(buf); { + if len(buf)-i < pl { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } txid := make([]byte, pl) - copy(txid, buf[pl*i:]) + copy(txid, buf[i:]) + i += pl + o, ol, err := d.unpackNOutpoints(buf[i:]) + if err != nil { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + bt = append(bt, blockTxs{ + btxID: txid, + inputs: o, + }) + i += ol } - return txids, nil + return bt, nil } func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { @@ -607,13 +642,11 @@ func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { func appendTxInput(txi *txInput, buf []byte, varBuf []byte) []byte { la := len(txi.addrID) - l := packVarint(la, varBuf) + l := packVaruint(uint(la), varBuf) buf = append(buf, varBuf[:l]...) buf = append(buf, txi.addrID...) l = packBigint(&txi.valueSat, varBuf) buf = append(buf, varBuf[:l]...) - l = packVaruint(uint(txi.vout), varBuf) - buf = append(buf, varBuf[:l]...) return buf } @@ -647,15 +680,12 @@ func unpackTxAddresses(buf []byte) (*txAddresses, error) { } func unpackTxInput(ti *txInput, buf []byte) int { - al, l := unpackVarint(buf) + al, l := unpackVaruint(buf) ti.addrID = make([]byte, al) - copy(ti.addrID, buf[l:l+al]) - al += l + copy(ti.addrID, buf[l:l+int(al)]) + al += uint(l) ti.valueSat, l = unpackBigint(buf[al:]) - al += l - v, l := unpackVaruint(buf[al:]) - ti.vout = uint32(v) - return l + al + return l + int(al) } func unpackTxOutput(to *txOutput, buf []byte) int { @@ -698,6 +728,26 @@ func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { return outpoints, nil } +func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + n, p := unpackVaruint(buf) + outpoints := make([]outpoint, n) + for i := uint(0); i < n; i++ { + if p+txidUnpackedLen >= len(buf) { + return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") + } + btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) + p += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[p:]) + p += voutLen + outpoints[i] = outpoint{ + btxID: btxID, + index: vout, + } + } + return outpoints, p, nil +} + func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { if len(addrID) > 0 { if len(addrID) > maxAddrIDLen { @@ -857,39 +907,7 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -// get all transactions of the given address and match it to input to find spent output -func (d *RocksDB) findSpentTx(ti *txInput) ([]byte, *txAddresses, error) { - start := packAddressKey(ti.addrID, 0) - stop := packAddressKey(ti.addrID, ^uint32(0)) - it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) - defer it.Close() - for it.Seek(start); it.Valid(); it.Next() { - key := it.Key().Data() - val := it.Value().Data() - if bytes.Compare(key, stop) > 0 { - break - } - outpoints, err := d.unpackOutpoints(val) - if err != nil { - return nil, nil, err - } - for _, o := range outpoints { - // process only outputs that match - if o.index >= 0 && uint32(o.index) == ti.vout { - a, err := d.getTxAddresses(o.btxID) - if err != nil { - return nil, nil, err - } - if bytes.Equal(a.outputs[o.index].addrID, ti.addrID) { - return o.btxID, a, nil - } - } - } - } - return nil, nil, nil -} - -func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *txAddresses, txAddressesToUpdate map[string]*txAddresses, balances map[string]*addrBalance) error { addresses := make(map[string]struct{}) getAddressBalance := func(addrID []byte) (*addrBalance, error) { @@ -905,46 +923,68 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } return b, nil } - for _, t := range txa.inputs { - s := string(t.addrID) - _, fa := addresses[s] - if !fa { - addresses[s] = struct{}{} - } - b, err := getAddressBalance(t.addrID) - if err != nil { - return err - } - if b != nil { - // subtract number of txs only once - if !fa { - b.txs-- + for i, t := range txa.inputs { + if len(t.addrID) > 0 { + s := string(t.addrID) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} } - b.sentSat.Sub(&b.sentSat, &t.valueSat) - if b.sentSat.Sign() < 0 { - d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + b, err := getAddressBalance(t.addrID) + if err != nil { + return err } - b.balanceSat.Add(&b.balanceSat, &t.valueSat) + if b != nil { + // subtract number of txs only once + if !exist { + b.txs-- + } + b.sentSat.Sub(&b.sentSat, &t.valueSat) + if b.sentSat.Sign() < 0 { + d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + } + b.balanceSat.Add(&b.balanceSat, &t.valueSat) + } else { + ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) + had := hex.EncodeToString(t.addrID) + glog.Warningf("Balance for address %s (%s) not found", ad, had) + } + s = string(inputs[i].btxID) + sa, exist := txAddressesToUpdate[s] + if !exist { + sa, err = d.getTxAddresses(inputs[i].btxID) + if err != nil { + return err + } + txAddressesToUpdate[s] = sa + } + sa.outputs[inputs[i].index].spent = false } } for _, t := range txa.outputs { - s := string(t.addrID) - _, fa := addresses[s] - if !fa { - addresses[s] = struct{}{} - } - b, err := getAddressBalance(t.addrID) - if err != nil { - return err - } - if b != nil { - // subtract number of txs only once - if !fa { - b.txs-- + if len(t.addrID) > 0 { + s := string(t.addrID) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} } - b.balanceSat.Sub(&b.balanceSat, &t.valueSat) - if b.balanceSat.Sign() < 0 { - d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + b, err := getAddressBalance(t.addrID) + if err != nil { + return err + } + if b != nil { + // subtract number of txs only once + if !exist { + b.txs-- + } + b.balanceSat.Sub(&b.balanceSat, &t.valueSat) + if b.balanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + } + } else { + ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) + had := hex.EncodeToString(t.addrID) + glog.Warningf("Balance for address %s (%s) not found", ad, had) } } } @@ -956,19 +996,19 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } // DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher -// if they are in the range kept in the cfBlockTxs column +// if they are in the range kept in the cfBlockTxids column func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - blocksTxids := make([][][]byte, higher-lower+1) + blocks := make([][]blockTxs, higher-lower+1) for height := lower; height <= higher; height++ { - blockTxids, err := d.getBlockTxids(height) + blockTxs, err := d.getBlockTxs(height) if err != nil { return err } - if len(blockTxids) == 0 { + if len(blockTxs) == 0 { return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) } - blocksTxids[height-lower] = blockTxids + blocks[height-lower] = blockTxs } wb := gorocksdb.NewWriteBatch() defer wb.Destroy() @@ -976,20 +1016,30 @@ func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { txsToDelete := make(map[string]struct{}) balances := make(map[string]*addrBalance) for height := higher; height >= lower; height-- { - blockTxids := blocksTxids[height-lower] - for _, txid := range blockTxids { + blockTxs := blocks[height-lower] + glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") + // go backwards to avoid interim negative balance + // when connecting block, amount is first in tx on the output side, then in another tx on the input side + // when disconnecting, it must be done backwards + for i := len(blockTxs) - 1; i >= 0; i-- { + txid := blockTxs[i].btxID + s := string(txid) + txsToDelete[s] = struct{}{} txa, err := d.getTxAddresses(txid) if err != nil { return err } - s := string(txid) - txsToDelete[s] = struct{}{} - if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, balances); err != nil { + if txa == nil { + ut, _ := d.chainParser.UnpackTxid(txid) + glog.Warning("TxAddress for txid ", ut, " not found") + continue + } + if err := d.disconnectTxAddresses(wb, height, s, blockTxs[i].inputs, txa, txAddressesToUpdate, balances); err != nil { return err } } key := packUint(height) - wb.DeleteCF(d.cfh[cfBlockTxids], key) + wb.DeleteCF(d.cfh[cfBlockTxs], key) wb.DeleteCF(d.cfh[cfHeight], key) } d.storeTxAddresses(wb, txAddressesToUpdate) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index fe841e82..7157931e 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -24,10 +24,9 @@ import ( // takes only 1 byte if abs(n)<127 func bitcoinTestnetParser() *btc.BitcoinParser { - return &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, - Params: btc.GetChainParams("test"), - } + return btc.NewBitcoinParser( + btc.GetChainParams("test"), + &btc.Configuration{BlockAddressesToKeep: 1}) } func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { @@ -62,6 +61,11 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { return hex.EncodeToString(b) } +func inputAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return strconv.FormatInt(int64(len(h)/2), 16) + h +} + func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) return strconv.FormatInt(int64(len(h)), 16) + h @@ -324,7 +328,7 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { } } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, }); err != nil { @@ -376,9 +380,21 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockTxids, []keyPair{ - keyPair{"000370d5", txidB1T1 + txidB1T2, nil}, - }); err != nil { + + var blockTxsKp []keyPair + if afterDisconnect { + blockTxsKp = []keyPair{} + } else { + blockTxsKp = []keyPair{ + keyPair{ + "000370d5", + txidB1T1 + "00" + txidB1T2 + "00", + nil, + }, + } + } + + if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil { { t.Fatal(err) } @@ -432,8 +448,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T1, "02" + - addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + "00" + - addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "01" + + inputAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + inputAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "02" + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), @@ -442,8 +458,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T2, "02" + - addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + "00" + - addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "01" + + inputAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + inputAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "02" + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), @@ -452,7 +468,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T3, "01" + - addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "02" + + inputAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "01" + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, @@ -477,8 +493,14 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockTxids, []keyPair{ - keyPair{"000370d6", txidB2T1 + txidB2T2 + txidB2T3, nil}, + if err := checkColumn(d, cfBlockTxs, []keyPair{ + keyPair{ + "000370d6", + txidB2T1 + "02" + txidB1T2 + "00" + txidB1T1 + "02" + + txidB2T2 + "02" + txidB2T1 + "00" + txidB1T2 + "02" + + txidB2T3 + "01" + txidB1T2 + "04", + nil, + }, }); err != nil { { t.Fatal(err) @@ -548,8 +570,8 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality // 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using BlockTxids column -// 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error +// 6) Disconnect the block 2 using BlockTxs column +// 7) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ @@ -562,7 +584,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d) + verifyAfterUTXOBlock1(t, d, false) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block block2 := getTestUTXOBlock2(t, d) @@ -645,14 +667,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { if err != nil { t.Fatal(err) } - - verifyAfterUTXOBlock1(t, d) + verifyAfterUTXOBlock1(t, d, true) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { t.Fatal(err) } } + // connect block again and verify the state of db + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + } func Test_packBigint_unpackBigint(t *testing.T) { @@ -762,18 +789,16 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }{ { name: "1", - hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b700e0392c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a19987283bb55012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + hex: "0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", data: &txAddresses{ inputs: []txInput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), valueSat: *big.NewInt(0), - vout: 12345, }, { addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), valueSat: *big.NewInt(1234123421342341234), - vout: 56789, }, }, outputs: []txOutput{ @@ -787,23 +812,20 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "2", - hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c0012ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c0022ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec003052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + hex: "0317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", data: &txAddresses{ inputs: []txInput{ { addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), valueSat: *big.NewInt(9011000000), - vout: 1, }, { addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), valueSat: *big.NewInt(8011000000), - vout: 2, }, { addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), valueSat: *big.NewInt(7011000000), - vout: 3, }, }, outputs: []txOutput{ @@ -835,13 +857,12 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "empty address", - hex: "01000204d201020002162e010162", + hex: "01000204d2020002162e010162", data: &txAddresses{ inputs: []txInput{ { addrID: []byte{}, valueSat: *big.NewInt(1234), - vout: 1, }, }, outputs: []txOutput{ From 878d25ea426c33ba59cac3a3c782ed8068966a13 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 17 Aug 2018 23:48:36 +0200 Subject: [PATCH 016/123] Fix handling of coinbase txs in db, add test for it --- db/rocksdb.go | 11 ++++++++-- db/rocksdb_test.go | 51 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 08d1eb78..74ebb3e1 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -515,8 +515,10 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa } func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { - buf := make([]byte, 0, d.chainParser.PackedTxidLen()*len(block.Txs)) + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, 0, pl*len(block.Txs)) varBuf := make([]byte, vlq.MaxLen64) + zeroTx := make([]byte, pl) for i := range block.Txs { tx := &block.Txs[i] o := make([]outpoint, len(tx.Vin)) @@ -524,7 +526,12 @@ func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchai vin := &tx.Vin[v] btxID, err := d.chainParser.PackTxid(vin.Txid) if err != nil { - return err + // do not process inputs without input txid + if err == bchain.ErrTxidMissing { + btxID = zeroTx + } else { + return err + } } o[v].btxID = btxID o[v].index = int32(vin.Vout) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 7157931e..3ffe7e11 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -12,7 +12,6 @@ import ( "os" "reflect" "sort" - "strconv" "strings" "testing" @@ -54,6 +53,9 @@ func closeAndDestroyRocksDB(t *testing.T, d *RocksDB) { } func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { + if addr == "" { + return "" + } b, err := d.chainParser.AddressToOutputScript(addr) if err != nil { t.Fatal(err) @@ -63,17 +65,17 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { func inputAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)/2), 16) + h + return hex.EncodeToString([]byte{byte(len(h) / 2)}) + h } func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)), 16) + h + return hex.EncodeToString([]byte{byte(len(h))}) + h } func spentAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)+1), 16) + h + return hex.EncodeToString([]byte{byte(len(h) + 1)}) + h } func bigintToHex(i *big.Int) string { @@ -143,6 +145,7 @@ const ( txidB2T1 = "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" txidB2T2 = "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" txidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + txidB2T4 = "fdd824a780cbb718eeb766eb05d83fdefc793a27082cd5e67f856d69798cf7db" addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac @@ -153,6 +156,7 @@ const ( addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac + addrA = "mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj" // 76a914d03c0d863d189b23b061a95ad32940b65837609f88ac ) var ( @@ -167,6 +171,7 @@ var ( satB2T2A8 = big.NewInt(118641975500) satB2T2A9 = big.NewInt(198641975500) satB2T3A5 = big.NewInt(9000) + satB2T4AA = big.NewInt(1360030331) ) func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { @@ -324,6 +329,31 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { Blocktime: 22549400002, Time: 22549400002, }, + // mining transaction + bchain.Tx{ + Txid: txidB2T4, + Vin: []bchain.Vin{ + bchain.Vin{ + Coinbase: "03bf1e1504aede765b726567696f6e312f50726f6a65637420425443506f6f6c2f01000001bf7e000000000000", + }, + }, + Vout: []bchain.Vout{ + bchain.Vout{ + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex(addrA, t, d), + }, + ValueSat: *satB2T4AA, + }, + bchain.Vout{ + N: 1, + ScriptPubKey: bchain.ScriptPubKey{}, + ValueSat: *satZero, + }, + }, + Blocktime: 22549400003, + Time: 22549400003, + }, }, } } @@ -423,6 +453,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{addressToPubKeyHex(addr3, t, d) + "000370d6", txidB2T1 + "01", nil}, keyPair{addressToPubKeyHex(addr2, t, d) + "000370d6", txidB2T1 + "03", nil}, keyPair{addressToPubKeyHex(addr5, t, d) + "000370d6", txidB2T3 + "00" + txidB2T3 + "01", nil}, + keyPair{addressToPubKeyHex(addrA, t, d) + "000370d6", txidB2T4 + "00", nil}, keyPair{addressToPubKeyHex(addr4, t, d) + "000370d6", txidB2T2 + "03", nil}, }); err != nil { { @@ -473,6 +504,14 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, }, + keyPair{ + txidB2T4, + "01" + inputAddressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero) + + "02" + + addressToPubKeyHexWithLength(addrA, t, d) + bigintToHex(satB2T4AA) + + addressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero), + nil, + }, }); err != nil { { t.Fatal(err) @@ -488,6 +527,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{addressToPubKeyHex(addr7, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T1A7), nil}, keyPair{addressToPubKeyHex(addr8, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A8), nil}, keyPair{addressToPubKeyHex(addr9, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A9), nil}, + keyPair{addressToPubKeyHex(addrA, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T4AA), nil}, }); err != nil { { t.Fatal(err) @@ -498,7 +538,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { "000370d6", txidB2T1 + "02" + txidB1T2 + "00" + txidB1T1 + "02" + txidB2T2 + "02" + txidB2T1 + "00" + txidB1T2 + "02" + - txidB2T3 + "01" + txidB1T2 + "04", + txidB2T3 + "01" + txidB1T2 + "04" + + txidB2T4 + "01" + "0000000000000000000000000000000000000000000000000000000000000000" + "00", nil, }, }); err != nil { From d45d028ef229cfe11875afd18a8d823b71aedc25 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 19 Aug 2018 00:23:26 +0200 Subject: [PATCH 017/123] Implement bulk connect blocks --- blockbook.go | 6 +- common/internalstate.go | 2 + db/rocksdb.go | 225 ++++++++++++++++++++++++++++++++++++---- db/rocksdb_test.go | 39 +++++++ db/sync.go | 15 ++- 5 files changed, 266 insertions(+), 21 deletions(-) diff --git a/blockbook.go b/blockbook.go index cea2c5b6..7e3da52b 100644 --- a/blockbook.go +++ b/blockbook.go @@ -175,7 +175,11 @@ func main() { } index.SetInternalState(internalState) if internalState.DbState != common.DbStateClosed { - glog.Warning("internalState: database in not closed state ", internalState.DbState, ", possibly previous ungraceful shutdown") + if internalState.DbState == common.DbStateInconsistent { + glog.Error("internalState: database is in inconsistent state and cannot be used") + return + } + glog.Warning("internalState: database was left in open state, possibly previous ungraceful shutdown") } if *computeColumnStats { diff --git a/common/internalstate.go b/common/internalstate.go index 8e7b2b61..1053649c 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -11,6 +11,8 @@ const ( DbStateClosed = uint32(iota) // DbStateOpen means db is open or application died without closing the db DbStateOpen + // DbStateInconsistent means db is in inconsistent state and cannot be used + DbStateInconsistent ) // InternalStateColumn contains the data of a db column diff --git a/db/rocksdb.go b/db/rocksdb.go index 74ebb3e1..267a0ff6 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -258,7 +258,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { // unspentTxs; therefore it is not possible to DisconnectBlocks this way return errors.New("DisconnectBlock is not supported for UTXO chains") } - if err := d.writeAddressesUTXO(wb, block); err != nil { + txAddressesMap := make(map[string]*txAddresses) + balances := make(map[string]*addrBalance) + if err := d.writeAddressesUTXO(wb, block, txAddressesMap, balances); err != nil { + return err + } + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { return err } } else { @@ -270,6 +281,177 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return d.db.Write(d.wo, wb) } +// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type BulkConnect struct { + d *RocksDB + isUTXO bool + txAddressesMap map[string]*txAddresses + balances map[string]*addrBalance + height uint32 +} + +const ( + maxBulkTxAddresses = 2000000 + partialStoreAddresses = maxBulkTxAddresses / 10 + maxBulkBalances = 2500000 + partialStoreBalances = maxBulkBalances / 10 +) + +func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { + bc := &BulkConnect{ + d: d, + isUTXO: d.chainParser.IsUTXOChain(), + txAddressesMap: make(map[string]*txAddresses), + balances: make(map[string]*addrBalance), + } + if err := d.SetInconsistentState(true); err != nil { + return nil, err + } + glog.Info("rocksdb: bulk connect init, db set to inconsistent state") + return bc, nil +} + +func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { + defer close(c) + start := time.Now() + var txm map[string]*txAddresses + if all { + txm = b.txAddressesMap + b.txAddressesMap = make(map[string]*txAddresses) + } else { + txm = make(map[string]*txAddresses) + for k, a := range b.txAddressesMap { + // store all completely spent transactions, they will not be modified again + r := true + for _, o := range a.outputs { + if o.spent == false { + r = false + break + } + } + if r { + txm[k] = a + delete(b.txAddressesMap, k) + } + } + // store some other random transactions if necessary + if len(txm) < partialStoreAddresses { + for k, a := range b.txAddressesMap { + txm[k] = a + delete(b.txAddressesMap, k) + if len(txm) >= partialStoreAddresses { + break + } + } + } + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.storeTxAddresses(wb, txm); err != nil { + c <- err + } else { + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + } + } + glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) +} + +func (b *BulkConnect) storeBalances(c chan error, all bool) { + defer close(c) + start := time.Now() + var bal map[string]*addrBalance + if all { + bal = b.balances + b.balances = make(map[string]*addrBalance) + } else { + bal = make(map[string]*addrBalance) + // store some random balances + for k, a := range b.balances { + bal[k] = a + delete(b.balances, k) + if len(bal) >= partialStoreBalances { + break + } + } + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.storeBalances(wb, bal); err != nil { + c <- err + } else { + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + } + } + glog.Info("rocksdb: height ", b.height, ", stored ", len(bal), " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) +} + +func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { + b.height = block.Height + if !b.isUTXO { + return b.d.ConnectBlock(block) + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.writeAddressesUTXO(wb, block, b.txAddressesMap, b.balances); err != nil { + return err + } + var storeAddressesChan, storeBalancesChan chan error + if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { + storeAddressesChan = make(chan error) + go b.storeTxAddresses(storeAddressesChan, false) + } + if len(b.balances)+partialStoreBalances > maxBulkBalances { + storeBalancesChan = make(chan error) + go b.storeBalances(storeBalancesChan, false) + } + } + if storeBlockTxs { + if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { + return err + } + } + if err := b.d.writeHeight(wb, block, opInsert); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + if storeAddressesChan != nil { + if err := <-storeAddressesChan; err != nil { + return err + } + } + if storeBalancesChan != nil { + if err := <-storeBalancesChan; err != nil { + return err + } + } + return nil +} + +func (b *BulkConnect) Close() error { + glog.Info("rocksdb: bulk connect closing") + storeAddressesChan := make(chan error) + go b.storeTxAddresses(storeAddressesChan, true) + storeBalancesChan := make(chan error) + go b.storeBalances(storeBalancesChan, true) + if err := <-storeAddressesChan; err != nil { + return err + } + if err := <-storeBalancesChan; err != nil { + return err + } + if err := b.d.SetInconsistentState(false); err != nil { + return err + } + glog.Info("rocksdb: bulk connect closed, db set to open state") + b.d = nil + return nil +} + // Addresses index type outpoint struct { @@ -315,11 +497,9 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText valueSat.SetInt64(0) } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { +func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { addresses := make(map[string][]outpoint) blockTxIDs := make([][]byte, len(block.Txs)) - txAddressesMap := make(map[string]*txAddresses) - balances := make(map[string]*addrBalance) // first process all outputs so that inputs can point to txs in this block for txi := range block.Txs { tx := &block.Txs[txi] @@ -454,16 +634,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } } - if err := d.storeAddresses(wb, block, addresses); err != nil { - return err - } - if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { - return err - } - if err := d.storeBalances(wb, balances); err != nil { - return err - } - return d.storeAndCleanupBlockTxs(wb, block) + return d.storeAddresses(wb, block, addresses) } func processedInTx(o []outpoint, btxID []byte) bool { @@ -1228,6 +1399,18 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro return is, nil } +func (d *RocksDB) SetInconsistentState(inconsistent bool) error { + if d.is == nil { + return errors.New("Internal state not created") + } + if inconsistent { + d.is.DbState = common.DbStateInconsistent + } else { + d.is.DbState = common.DbStateOpen + } + return d.storeState(d.is) +} + // SetInternalState sets the InternalState to be used by db to collect internal state func (d *RocksDB) SetInternalState(is *common.InternalState) { d.is = is @@ -1235,11 +1418,17 @@ func (d *RocksDB) SetInternalState(is *common.InternalState) { // StoreInternalState stores the internal state to db func (d *RocksDB) StoreInternalState(is *common.InternalState) error { - for c := 0; c < len(cfNames); c++ { - rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) - d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) - d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + if d.metrics != nil { + for c := 0; c < len(cfNames); c++ { + rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) + d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) + d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + } } + return d.storeState(is) +} + +func (d *RocksDB) storeState(is *common.InternalState) error { buf, err := is.Pack() if err != nil { return err diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 3ffe7e11..adf9e6c9 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -5,6 +5,7 @@ package db import ( "blockbook/bchain" "blockbook/bchain/coins/btc" + "blockbook/common" "encoding/hex" "fmt" "io/ioutil" @@ -720,7 +721,45 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } verifyAfterUTXOBlock2(t, d) +} +func Test_BulkConnect_UTXO(t *testing.T) { + d := setupRocksDB(t, &testBitcoinParser{ + BitcoinParser: bitcoinTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + bc, err := d.InitBulkConnect() + if err != nil { + t.Fatal(err) + } + + if d.is.DbState != common.DbStateInconsistent { + t.Fatal("DB not in DbStateInconsistent") + } + + if err := bc.ConnectBlock(getTestUTXOBlock1(t, d), false); err != nil { + t.Fatal(err) + } + if err := checkColumn(d, cfBlockTxs, []keyPair{}); err != nil { + { + t.Fatal(err) + } + } + + if err := bc.ConnectBlock(getTestUTXOBlock2(t, d), true); err != nil { + t.Fatal(err) + } + + if err := bc.Close(); err != nil { + t.Fatal(err) + } + + if d.is.DbState != common.DbStateOpen { + t.Fatal("DB not in DbStateOpen") + } + + verifyAfterUTXOBlock2(t, d) } func Test_packBigint_unpackBigint(t *testing.T) { diff --git a/db/sync.go b/db/sync.go index fecfa09c..32bc143d 100644 --- a/db/sync.go +++ b/db/sync.go @@ -213,17 +213,26 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { writeBlockDone := make(chan struct{}) writeBlockWorker := func() { defer close(writeBlockDone) + bc, err := w.db.InitBulkConnect() + if err != nil { + glog.Error("sync: InitBulkConnect error ", err) + } lastBlock := lower - 1 + keep := uint32(w.chain.GetChainParser().KeepBlockAddresses()) for b := range bch { if lastBlock+1 != b.Height { glog.Error("writeBlockWorker skipped block, last connected block", lastBlock, ", new block ", b.Height) } - err := w.db.ConnectBlock(b) + err := bc.ConnectBlock(b, b.Height+keep > higher) if err != nil { glog.Error("writeBlockWorker ", b.Height, " ", b.Hash, " error ", err) } lastBlock = b.Height } + err = bc.Close() + if err != nil { + glog.Error("sync: bulkconnect.Close error ", err) + } glog.Info("WriteBlock exiting...") } getBlockWorker := func(i int) { @@ -276,6 +285,7 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { } go writeBlockWorker() var hash string + start := time.Now() ConnectLoop: for h := lower; h <= higher; { select { @@ -292,7 +302,8 @@ ConnectLoop: } hch <- hashHeight{hash, h} if h > 0 && h%1000 == 0 { - glog.Info("connecting block ", h, " ", hash) + glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start)) + start = time.Now() } h++ } From 5621ed49f3b642adf996c2d8dc1d1011bd526d89 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 19 Aug 2018 17:43:00 +0200 Subject: [PATCH 018/123] Optimize bulk connect of blocks --- db/rocksdb.go | 79 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 267a0ff6..1141df55 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -258,9 +258,13 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { // unspentTxs; therefore it is not possible to DisconnectBlocks this way return errors.New("DisconnectBlock is not supported for UTXO chains") } + addresses := make(map[string][]outpoint) txAddressesMap := make(map[string]*txAddresses) balances := make(map[string]*addrBalance) - if err := d.writeAddressesUTXO(wb, block, txAddressesMap, balances); err != nil { + if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { + return err + } + if err := d.storeAddresses(wb, block.Height, addresses); err != nil { return err } if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { @@ -282,15 +286,23 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { } // BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type bulkAddresses struct { + height uint32 + addresses map[string][]outpoint +} + type BulkConnect struct { - d *RocksDB - isUTXO bool - txAddressesMap map[string]*txAddresses - balances map[string]*addrBalance - height uint32 + d *RocksDB + isUTXO bool + bulkAddresses []bulkAddresses + bulkAddressesCount int + txAddressesMap map[string]*txAddresses + balances map[string]*addrBalance + height uint32 } const ( + maxBulkAddresses = 400000 maxBulkTxAddresses = 2000000 partialStoreAddresses = maxBulkTxAddresses / 10 maxBulkBalances = 2500000 @@ -315,6 +327,7 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { defer close(c) start := time.Now() var txm map[string]*txAddresses + var sp int if all { txm = b.txAddressesMap b.txAddressesMap = make(map[string]*txAddresses) @@ -334,6 +347,7 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { delete(b.txAddressesMap, k) } } + sp = len(txm) // store some other random transactions if necessary if len(txm) < partialStoreAddresses { for k, a := range b.txAddressesMap { @@ -354,7 +368,7 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { c <- err } } - glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) } func (b *BulkConnect) storeBalances(c chan error, all bool) { @@ -387,6 +401,17 @@ func (b *BulkConnect) storeBalances(c chan error, all bool) { glog.Info("rocksdb: height ", b.height, ", stored ", len(bal), " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) } +func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { + for _, ba := range b.bulkAddresses { + if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { + return err + } + } + b.bulkAddressesCount = 0 + b.bulkAddresses = b.bulkAddresses[:0] + return nil +} + func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { b.height = block.Height if !b.isUTXO { @@ -394,11 +419,15 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro } wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - if err := b.d.writeAddressesUTXO(wb, block, b.txAddressesMap, b.balances); err != nil { + addresses := make(map[string][]outpoint) + if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { return err } + start := time.Now() + var sa bool var storeAddressesChan, storeBalancesChan chan error if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + sa = true if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { storeAddressesChan = make(chan error) go b.storeTxAddresses(storeAddressesChan, false) @@ -408,6 +437,17 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro go b.storeBalances(storeBalancesChan, false) } } + b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ + height: block.Height, + addresses: addresses, + }) + b.bulkAddressesCount += len(addresses) + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + } if storeBlockTxs { if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { return err @@ -419,6 +459,9 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro if err := b.d.db.Write(b.d.wo, wb); err != nil { return err } + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + } if storeAddressesChan != nil { if err := <-storeAddressesChan; err != nil { return err @@ -434,10 +477,21 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro func (b *BulkConnect) Close() error { glog.Info("rocksdb: bulk connect closing") + start := time.Now() storeAddressesChan := make(chan error) go b.storeTxAddresses(storeAddressesChan, true) storeBalancesChan := make(chan error) go b.storeBalances(storeBalancesChan, true) + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) if err := <-storeAddressesChan; err != nil { return err } @@ -497,8 +551,7 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText valueSat.SetInt64(0) } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { - addresses := make(map[string][]outpoint) +func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { blockTxIDs := make([][]byte, len(block.Txs)) // first process all outputs so that inputs can point to txs in this block for txi := range block.Txs { @@ -634,7 +687,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } } - return d.storeAddresses(wb, block, addresses) + return nil } func processedInTx(o []outpoint, btxID []byte) bool { @@ -646,10 +699,10 @@ func processedInTx(o []outpoint, btxID []byte) bool { return false } -func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, block *bchain.Block, addresses map[string][]outpoint) error { +func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error { for addrID, outpoints := range addresses { ba := []byte(addrID) - key := packAddressKey(ba, block.Height) + key := packAddressKey(ba, height) val := d.packOutpoints(outpoints) wb.PutCF(d.cfh[cfAddresses], key, val) } From ab53107f47855bb41301b458b6eaac9355bc1419 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 19 Aug 2018 23:53:28 +0200 Subject: [PATCH 019/123] Adapt monacoin to bigint amounts --- bchain/coins/monacoin/monacoin_test.go | 9 +++++---- bchain/coins/monacoin/monacoinrpc.go | 6 +----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/bchain/coins/monacoin/monacoin_test.go b/bchain/coins/monacoin/monacoin_test.go index 207f3378..6e41db33 100644 --- a/bchain/coins/monacoin/monacoin_test.go +++ b/bchain/coins/monacoin/monacoin_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -173,8 +174,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.21420790, - N: 0, + ValueSat: *big.NewInt(21420790), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914fb69fe6dcfe88557dc0ce0ea65bd7cf02f5e4f5b88ac", Addresses: []string{ @@ -184,8 +185,8 @@ func init() { Address: addr1, }, { - Value: 14.68857739, - N: 1, + ValueSat: *big.NewInt(1468857739), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914628d603ac50d656e3311ff0cd5490b4c5cdd92ea88ac", Addresses: []string{ diff --git a/bchain/coins/monacoin/monacoinrpc.go b/bchain/coins/monacoin/monacoinrpc.go index 7a522a34..dacddb4c 100644 --- a/bchain/coins/monacoin/monacoinrpc.go +++ b/bchain/coins/monacoin/monacoinrpc.go @@ -24,6 +24,7 @@ func NewMonacoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *MonacoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *MonacoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} From c67306ad09f99bbf3b74cd2f74e7967adfd76131 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 20 Aug 2018 18:35:46 +0200 Subject: [PATCH 020/123] Make AddrBalance and TxAddresses publicly loadable from DB --- db/rocksdb.go | 242 ++++++++++++++++++++++++++------------------- db/rocksdb_test.go | 126 ++++++++++++++++------- 2 files changed, 231 insertions(+), 137 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 1141df55..5cd59268 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -259,8 +259,8 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return errors.New("DisconnectBlock is not supported for UTXO chains") } addresses := make(map[string][]outpoint) - txAddressesMap := make(map[string]*txAddresses) - balances := make(map[string]*addrBalance) + txAddressesMap := make(map[string]*TxAddresses) + balances := make(map[string]*AddrBalance) if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { return err } @@ -296,8 +296,8 @@ type BulkConnect struct { isUTXO bool bulkAddresses []bulkAddresses bulkAddressesCount int - txAddressesMap map[string]*txAddresses - balances map[string]*addrBalance + txAddressesMap map[string]*TxAddresses + balances map[string]*AddrBalance height uint32 } @@ -313,8 +313,8 @@ func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { bc := &BulkConnect{ d: d, isUTXO: d.chainParser.IsUTXOChain(), - txAddressesMap: make(map[string]*txAddresses), - balances: make(map[string]*addrBalance), + txAddressesMap: make(map[string]*TxAddresses), + balances: make(map[string]*AddrBalance), } if err := d.SetInconsistentState(true); err != nil { return nil, err @@ -326,18 +326,18 @@ func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { defer close(c) start := time.Now() - var txm map[string]*txAddresses + var txm map[string]*TxAddresses var sp int if all { txm = b.txAddressesMap - b.txAddressesMap = make(map[string]*txAddresses) + b.txAddressesMap = make(map[string]*TxAddresses) } else { - txm = make(map[string]*txAddresses) + txm = make(map[string]*TxAddresses) for k, a := range b.txAddressesMap { // store all completely spent transactions, they will not be modified again r := true - for _, o := range a.outputs { - if o.spent == false { + for _, o := range a.Outputs { + if o.Spent == false { r = false break } @@ -374,12 +374,12 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { func (b *BulkConnect) storeBalances(c chan error, all bool) { defer close(c) start := time.Now() - var bal map[string]*addrBalance + var bal map[string]*AddrBalance if all { bal = b.balances - b.balances = make(map[string]*addrBalance) + b.balances = make(map[string]*AddrBalance) } else { - bal = make(map[string]*addrBalance) + bal = make(map[string]*AddrBalance) // store some random balances for k, a := range b.balances { bal[k] = a @@ -513,26 +513,42 @@ type outpoint struct { index int32 } -type txInput struct { +type TxInput struct { addrID []byte - valueSat big.Int + ValueSat big.Int } -type txOutput struct { +func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, error) { + // TODO - we will need AddressesFromAddrID parser method, this will not work for ZCash + return p.OutputScriptToAddresses(ti.addrID) +} + +type TxOutput struct { addrID []byte - spent bool - valueSat big.Int + Spent bool + ValueSat big.Int } -type txAddresses struct { - inputs []txInput - outputs []txOutput +func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, error) { + // TODO - we will need AddressesFromAddrID parser method, this will not work for ZCash + return p.OutputScriptToAddresses(to.addrID) } -type addrBalance struct { - txs uint32 - sentSat big.Int - balanceSat big.Int +type TxAddresses struct { + Inputs []TxInput + Outputs []TxOutput +} + +type AddrBalance struct { + Txs uint32 + SentSat big.Int + BalanceSat big.Int +} + +func (ab *AddrBalance) ReceivedSat() big.Int { + var r big.Int + r.Add(&ab.BalanceSat, &ab.SentSat) + return r } type blockTxs struct { @@ -551,7 +567,7 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText valueSat.SetInt64(0) } -func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { +func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { blockTxIDs := make([][]byte, len(block.Txs)) // first process all outputs so that inputs can point to txs in this block for txi := range block.Txs { @@ -561,12 +577,12 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string return err } blockTxIDs[txi] = btxID - ta := txAddresses{} - ta.outputs = make([]txOutput, len(tx.Vout)) + ta := TxAddresses{} + ta.Outputs = make([]TxOutput, len(tx.Vout)) txAddressesMap[string(btxID)] = &ta for i, output := range tx.Vout { - tao := &ta.outputs[i] - tao.valueSat = output.ValueSat + tao := &ta.Outputs[i] + tao.ValueSat = output.ValueSat addrID, err := d.chainParser.GetAddrIDFromVout(&output) if err != nil || len(addrID) == 0 || len(addrID) > maxAddrIDLen { if err != nil { @@ -593,20 +609,20 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string }) ab, e := balances[strAddrID] if !e { - ab, err = d.getAddressBalance(addrID) + ab, err = d.getAddrIDBalance(addrID) if err != nil { return err } if ab == nil { - ab = &addrBalance{} + ab = &AddrBalance{} } balances[strAddrID] = ab } // add number of trx in balance only once, address can be multiple times in tx if !processed { - ab.txs++ + ab.Txs++ } - ab.balanceSat.Add(&ab.balanceSat, &output.ValueSat) + ab.BalanceSat.Add(&ab.BalanceSat, &output.ValueSat) } } // process inputs @@ -614,9 +630,9 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string tx := &block.Txs[txi] spendingTxid := blockTxIDs[txi] ta := txAddressesMap[string(spendingTxid)] - ta.inputs = make([]txInput, len(tx.Vin)) + ta.Inputs = make([]TxInput, len(tx.Vin)) for i, input := range tx.Vin { - tai := &ta.inputs[i] + tai := &ta.Inputs[i] btxID, err := d.chainParser.PackTxid(input.Txid) if err != nil { // do not process inputs without input txid @@ -638,18 +654,18 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string } txAddressesMap[stxID] = ita } - if len(ita.outputs) <= int(input.Vout) { + if len(ita.Outputs) <= int(input.Vout) { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) continue } - ot := &ita.outputs[int(input.Vout)] - if ot.spent { + ot := &ita.Outputs[int(input.Vout)] + if ot.Spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } tai.addrID = ot.addrID - tai.valueSat = ot.valueSat + tai.ValueSat = ot.ValueSat // mark the output as spent in tx - ot.spent = true + ot.Spent = true if len(ot.addrID) == 0 { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) continue @@ -667,24 +683,24 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string }) ab, e := balances[strAddrID] if !e { - ab, err = d.getAddressBalance(ot.addrID) + ab, err = d.getAddrIDBalance(ot.addrID) if err != nil { return err } if ab == nil { - ab = &addrBalance{} + ab = &AddrBalance{} } balances[strAddrID] = ab } // add number of trx in balance only once, address can be multiple times in tx if !processed { - ab.txs++ + ab.Txs++ } - ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) - if ab.balanceSat.Sign() < 0 { - d.resetValueSatToZero(&ab.balanceSat, ot.addrID, "balance") + ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat) + if ab.BalanceSat.Sign() < 0 { + d.resetValueSatToZero(&ab.BalanceSat, ot.addrID, "balance") } - ab.sentSat.Add(&ab.sentSat, &ot.valueSat) + ab.SentSat.Add(&ab.SentSat, &ot.ValueSat) } } return nil @@ -709,7 +725,7 @@ func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addres return nil } -func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAddresses) error { +func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*TxAddresses) error { varBuf := make([]byte, maxPackedBigintBytes) buf := make([]byte, 1024) for txID, ta := range am { @@ -719,18 +735,18 @@ func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAd return nil } -func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBalance) error { +func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBalance) error { // allocate buffer big enough for number of txs + 2 bigints buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) for addrID, ab := range abm { // balance with 0 transactions is removed from db - happens in disconnect - if ab == nil || ab.txs <= 0 { + if ab == nil || ab.Txs <= 0 { wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrID)) } else { - l := packVaruint(uint(ab.txs), buf) - ll := packBigint(&ab.sentSat, buf[l:]) + l := packVaruint(uint(ab.Txs), buf) + ll := packBigint(&ab.SentSat, buf[l:]) l += ll - ll = packBigint(&ab.balanceSat, buf[l:]) + ll = packBigint(&ab.BalanceSat, buf[l:]) l += ll wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) } @@ -821,7 +837,7 @@ func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { return bt, nil } -func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { +func (d *RocksDB) getAddrIDBalance(addrID []byte) (*AddrBalance, error) { val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) if err != nil { return nil, err @@ -835,14 +851,23 @@ func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { txs, l := unpackVaruint(buf) sentSat, sl := unpackBigint(buf[l:]) balanceSat, _ := unpackBigint(buf[l+sl:]) - return &addrBalance{ - txs: uint32(txs), - sentSat: sentSat, - balanceSat: balanceSat, + return &AddrBalance{ + Txs: uint32(txs), + SentSat: sentSat, + BalanceSat: balanceSat, }, nil } -func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { +// GetAddressBalance returns address balance for an address or nil if address not found +func (d *RocksDB) GetAddressBalance(address string) (*AddrBalance, error) { + addrID, err := d.chainParser.GetAddrIDFromAddress(address) + if err != nil { + return nil, err + } + return d.getAddrIDBalance(addrID) +} + +func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) if err != nil { return nil, err @@ -856,79 +881,88 @@ func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { return unpackTxAddresses(buf) } -func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { - buf = buf[:0] - l := packVaruint(uint(len(ta.inputs)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ta.inputs { - buf = appendTxInput(&ta.inputs[i], buf, varBuf) +// GetTxAddresses returns TxAddresses for given txid or nil if not found +func (d *RocksDB) GetTxAddresses(txid string) (*TxAddresses, error) { + btxID, err := d.chainParser.PackTxid(txid) + if err != nil { + return nil, err } - l = packVaruint(uint(len(ta.outputs)), varBuf) + return d.getTxAddresses(btxID) +} + +func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { + buf = buf[:0] + l := packVaruint(uint(len(ta.Inputs)), varBuf) buf = append(buf, varBuf[:l]...) - for i := range ta.outputs { - buf = appendTxOutput(&ta.outputs[i], buf, varBuf) + for i := range ta.Inputs { + buf = appendTxInput(&ta.Inputs[i], buf, varBuf) + } + l = packVaruint(uint(len(ta.Outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.Outputs { + buf = appendTxOutput(&ta.Outputs[i], buf, varBuf) } return buf } -func appendTxInput(txi *txInput, buf []byte, varBuf []byte) []byte { +func appendTxInput(txi *TxInput, buf []byte, varBuf []byte) []byte { la := len(txi.addrID) l := packVaruint(uint(la), varBuf) buf = append(buf, varBuf[:l]...) buf = append(buf, txi.addrID...) - l = packBigint(&txi.valueSat, varBuf) + l = packBigint(&txi.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf } -func appendTxOutput(txo *txOutput, buf []byte, varBuf []byte) []byte { +func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { la := len(txo.addrID) - if txo.spent { + if txo.Spent { la = ^la } l := packVarint(la, varBuf) buf = append(buf, varBuf[:l]...) buf = append(buf, txo.addrID...) - l = packBigint(&txo.valueSat, varBuf) + l = packBigint(&txo.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf } -func unpackTxAddresses(buf []byte) (*txAddresses, error) { - ta := txAddresses{} +func unpackTxAddresses(buf []byte) (*TxAddresses, error) { + ta := TxAddresses{} inputs, l := unpackVaruint(buf) - ta.inputs = make([]txInput, inputs) + ta.Inputs = make([]TxInput, inputs) for i := uint(0); i < inputs; i++ { - l += unpackTxInput(&ta.inputs[i], buf[l:]) + l += unpackTxInput(&ta.Inputs[i], buf[l:]) } outputs, ll := unpackVaruint(buf[l:]) l += ll - ta.outputs = make([]txOutput, outputs) + ta.Outputs = make([]TxOutput, outputs) for i := uint(0); i < outputs; i++ { - l += unpackTxOutput(&ta.outputs[i], buf[l:]) + l += unpackTxOutput(&ta.Outputs[i], buf[l:]) } return &ta, nil } -func unpackTxInput(ti *txInput, buf []byte) int { +func unpackTxInput(ti *TxInput, buf []byte) int { al, l := unpackVaruint(buf) ti.addrID = make([]byte, al) copy(ti.addrID, buf[l:l+int(al)]) al += uint(l) - ti.valueSat, l = unpackBigint(buf[al:]) + ti.ValueSat, l = unpackBigint(buf[al:]) return l + int(al) } -func unpackTxOutput(to *txOutput, buf []byte) int { +func unpackTxOutput(to *TxOutput, buf []byte) int { al, l := unpackVarint(buf) if al < 0 { - to.spent = true + to.Spent = true al = ^al } to.addrID = make([]byte, al) copy(to.addrID, buf[l:l+al]) al += l - to.valueSat, l = unpackBigint(buf[al:]) + to.ValueSat, l = unpackBigint(buf[al:]) return l + al } @@ -1138,15 +1172,15 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *txAddresses, - txAddressesToUpdate map[string]*txAddresses, balances map[string]*addrBalance) error { +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *TxAddresses, + txAddressesToUpdate map[string]*TxAddresses, balances map[string]*AddrBalance) error { addresses := make(map[string]struct{}) - getAddressBalance := func(addrID []byte) (*addrBalance, error) { + getAddressBalance := func(addrID []byte) (*AddrBalance, error) { var err error s := string(addrID) b, fb := balances[s] if !fb { - b, err = d.getAddressBalance(addrID) + b, err = d.getAddrIDBalance(addrID) if err != nil { return nil, err } @@ -1154,7 +1188,7 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } return b, nil } - for i, t := range txa.inputs { + for i, t := range txa.Inputs { if len(t.addrID) > 0 { s := string(t.addrID) _, exist := addresses[s] @@ -1168,13 +1202,13 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, if b != nil { // subtract number of txs only once if !exist { - b.txs-- + b.Txs-- } - b.sentSat.Sub(&b.sentSat, &t.valueSat) - if b.sentSat.Sign() < 0 { - d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + b.SentSat.Sub(&b.SentSat, &t.ValueSat) + if b.SentSat.Sign() < 0 { + d.resetValueSatToZero(&b.SentSat, t.addrID, "sent amount") } - b.balanceSat.Add(&b.balanceSat, &t.valueSat) + b.BalanceSat.Add(&b.BalanceSat, &t.ValueSat) } else { ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) had := hex.EncodeToString(t.addrID) @@ -1189,10 +1223,10 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } txAddressesToUpdate[s] = sa } - sa.outputs[inputs[i].index].spent = false + sa.Outputs[inputs[i].index].Spent = false } } - for _, t := range txa.outputs { + for _, t := range txa.Outputs { if len(t.addrID) > 0 { s := string(t.addrID) _, exist := addresses[s] @@ -1206,11 +1240,11 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, if b != nil { // subtract number of txs only once if !exist { - b.txs-- + b.Txs-- } - b.balanceSat.Sub(&b.balanceSat, &t.valueSat) - if b.balanceSat.Sign() < 0 { - d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + b.BalanceSat.Sub(&b.BalanceSat, &t.ValueSat) + if b.BalanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.BalanceSat, t.addrID, "balance") } } else { ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) @@ -1243,9 +1277,9 @@ func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { } wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - txAddressesToUpdate := make(map[string]*txAddresses) + txAddressesToUpdate := make(map[string]*TxAddresses) txsToDelete := make(map[string]struct{}) - balances := make(map[string]*addrBalance) + balances := make(map[string]*AddrBalance) for height := higher; height >= lower; height-- { blockTxs := blocks[height-lower] glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index adf9e6c9..cd33f6ed 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -721,6 +721,66 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } verifyAfterUTXOBlock2(t, d) + + // test public methods for address balance and tx addresses + + ab, err := d.GetAddressBalance(addr5) + if err != nil { + t.Fatal(err) + } + abw := &AddrBalance{ + Txs: 2, + SentSat: *satB1T2A5, + BalanceSat: *satB2T3A5, + } + if !reflect.DeepEqual(ab, abw) { + t.Errorf("GetAddressBalance() = %+v, want %+v", ab, abw) + } + rs := ab.ReceivedSat() + rsw := satB1T2A5.Add(satB1T2A5, satB2T3A5) + if rs.Cmp(rsw) != 0 { + t.Errorf("GetAddressBalance().ReceivedSat() = %v, want %v", rs, rsw) + } + + ta, err := d.GetTxAddresses(txidB2T1) + if err != nil { + t.Fatal(err) + } + taw := &TxAddresses{ + Inputs: []TxInput{ + { + addrID: addressToOutput(addr3, d.chainParser), + ValueSat: *satB1T2A3, + }, + { + addrID: addressToOutput(addr2, d.chainParser), + ValueSat: *satB1T1A2, + }, + }, + Outputs: []TxOutput{ + { + addrID: addressToOutput(addr6, d.chainParser), + Spent: true, + ValueSat: *satB2T1A6, + }, + { + addrID: addressToOutput(addr7, d.chainParser), + Spent: false, + ValueSat: *satB2T1A7, + }, + }, + } + if !reflect.DeepEqual(ta, taw) { + t.Errorf("GetTxAddresses() = %+v, want %+v", ta, taw) + } + ia, err := ta.Inputs[0].Addresses(d.chainParser) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(ia, []string{addr3}) { + t.Errorf("GetTxAddresses().Inputs[0].Addresses() = %v, want %v", ia, []string{addr3}) + } + } func Test_BulkConnect_UTXO(t *testing.T) { @@ -852,7 +912,7 @@ func Test_packBigint_unpackBigint(t *testing.T) { } } -func addressToOutput(addr string, parser *btc.BitcoinParser) []byte { +func addressToOutput(addr string, parser bchain.BlockChainParser) []byte { b, err := parser.AddressToOutputScript(addr) if err != nil { panic(err) @@ -865,27 +925,27 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { tests := []struct { name string hex string - data *txAddresses + data *TxAddresses }{ { name: "1", hex: "0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", - data: &txAddresses{ - inputs: []txInput{ + data: &TxAddresses{ + Inputs: []TxInput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), - valueSat: *big.NewInt(0), + ValueSat: *big.NewInt(0), }, { addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), - valueSat: *big.NewInt(1234123421342341234), + ValueSat: *big.NewInt(1234123421342341234), }, }, - outputs: []txOutput{ + Outputs: []TxOutput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), - valueSat: *big.NewInt(1), - spent: true, + ValueSat: *big.NewInt(1), + Spent: true, }, }, }, @@ -893,44 +953,44 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { { name: "2", hex: "0317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", - data: &txAddresses{ - inputs: []txInput{ + data: &TxAddresses{ + Inputs: []TxInput{ { addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), - valueSat: *big.NewInt(9011000000), + ValueSat: *big.NewInt(9011000000), }, { addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), - valueSat: *big.NewInt(8011000000), + ValueSat: *big.NewInt(8011000000), }, { addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), - valueSat: *big.NewInt(7011000000), + ValueSat: *big.NewInt(7011000000), }, }, - outputs: []txOutput{ + Outputs: []TxOutput{ { addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), - valueSat: *big.NewInt(5011000000), - spent: true, + ValueSat: *big.NewInt(5011000000), + Spent: true, }, { addrID: addressToOutput("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), - valueSat: *big.NewInt(6011000000), + ValueSat: *big.NewInt(6011000000), }, { addrID: addressToOutput("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), - valueSat: *big.NewInt(7011000000), - spent: true, + ValueSat: *big.NewInt(7011000000), + Spent: true, }, { addrID: addressToOutput("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), - valueSat: *big.NewInt(999900000), + ValueSat: *big.NewInt(999900000), }, { addrID: addressToOutput("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), - valueSat: *big.NewInt(5000000000), - spent: true, + ValueSat: *big.NewInt(5000000000), + Spent: true, }, }, }, @@ -938,22 +998,22 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { { name: "empty address", hex: "01000204d2020002162e010162", - data: &txAddresses{ - inputs: []txInput{ + data: &TxAddresses{ + Inputs: []TxInput{ { addrID: []byte{}, - valueSat: *big.NewInt(1234), + ValueSat: *big.NewInt(1234), }, }, - outputs: []txOutput{ + Outputs: []TxOutput{ { addrID: []byte{}, - valueSat: *big.NewInt(5678), + ValueSat: *big.NewInt(5678), }, { addrID: []byte{}, - valueSat: *big.NewInt(98), - spent: true, + ValueSat: *big.NewInt(98), + Spent: true, }, }, }, @@ -961,9 +1021,9 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { { name: "empty", hex: "0000", - data: &txAddresses{ - inputs: []txInput{}, - outputs: []txOutput{}, + data: &TxAddresses{ + Inputs: []TxInput{}, + Outputs: []TxOutput{}, }, }, } From bbc47db6dd68e5c482a69ea221c59cad0f66589c Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 21 Aug 2018 10:11:27 +0200 Subject: [PATCH 021/123] Get address data for explorer using index v3 - WIP --- api/types.go | 3 ++ api/worker.go | 81 ++++++++++++++++++++++++++++++++++++++++++++++-- db/rocksdb.go | 4 +-- server/public.go | 11 ++++--- 4 files changed, 90 insertions(+), 9 deletions(-) diff --git a/api/types.go b/api/types.go index 287e04af..be289bd6 100644 --- a/api/types.go +++ b/api/types.go @@ -61,4 +61,7 @@ type Address struct { UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` TxApperances int `json:"txApperances"` Transactions []*Tx `json:"transactions"` + Page int `json:"page"` + TotalPages int `json:"totalPages"` + TxsOnPage int `json:"txsOnPage"` } diff --git a/api/worker.go b/api/worker.go index 01064560..64a93bff 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,13 +4,12 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "errors" "math/big" "github.com/golang/glog" ) -const txsOnPage = 30 - // Worker is handle to api worker type Worker struct { db *db.RocksDB @@ -172,7 +171,83 @@ func UniqueTxidsInReverse(txids []string) []string { } // GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { +func (w *Worker) GetAddressNew(address string, page int, txsOnPage int) (*Address, error) { + glog.Info(address, " start") + ba, err := w.db.GetAddressBalance(address) + if err != nil { + return nil, err + } + if ba == nil { + return nil, errors.New("Address not found") + } + txc, err := w.getAddressTxids(address, false) + txc = UniqueTxidsInReverse(txc) + if err != nil { + return nil, err + } + txm, err := w.getAddressTxids(address, true) + if err != nil { + return nil, err + } + txm = UniqueTxidsInReverse(txm) + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return nil, err + } + // paging + if page < 0 { + page = 0 + } + from := page * txsOnPage + totalPages := len(txc) / txsOnPage + if from >= len(txc) { + page = totalPages - 1 + if page < 0 { + page = 0 + } + } + from = page * txsOnPage + to := (page + 1) * txsOnPage + if to > len(txc) { + to = len(txc) + } + txs := make([]*Tx, len(txm)+to-from) + txi := 0 + // load mempool transactions + var uBalSat big.Int + for _, tx := range txm { + tx, err := w.GetTransaction(tx, bestheight, false) + // mempool transaction may fail + if err != nil { + glog.Error("GetTransaction ", tx, ": ", err) + } else { + uBalSat.Sub(tx.getAddrVoutValue(address), tx.getAddrVinValue(address)) + txs[txi] = tx + txi++ + } + } + if len(txc) != int(ba.Txs) { + glog.Warning("DB inconsistency in address ", address, ": number of txs from column addresses ", len(txc), ", from addressBalance ", ba.Txs) + } + + r := &Address{ + AddrStr: address, + Balance: w.chainParser.AmountToDecimalString(&ba.BalanceSat), + TotalReceived: w.chainParser.AmountToDecimalString(ba.ReceivedSat()), + TotalSent: w.chainParser.AmountToDecimalString(&ba.SentSat), + TxApperances: len(txc), + UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), + UnconfirmedTxApperances: len(txm), + Page: page, + TotalPages: totalPages, + TxsOnPage: txsOnPage, + } + glog.Info(address, " finished") + return r, nil +} + +// GetAddress computes address value and gets transactions for given address +func (w *Worker) GetAddress(addrID string, page int, txsOnPage int) (*Address, error) { glog.Info(addrID, " start") txc, err := w.getAddressTxids(addrID, false) txc = UniqueTxidsInReverse(txc) diff --git a/db/rocksdb.go b/db/rocksdb.go index 5cd59268..ef3be7db 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -545,10 +545,10 @@ type AddrBalance struct { BalanceSat big.Int } -func (ab *AddrBalance) ReceivedSat() big.Int { +func (ab *AddrBalance) ReceivedSat() *big.Int { var r big.Int r.Add(&ab.BalanceSat, &ab.SentSat) - return r + return &r } type blockTxs struct { diff --git a/server/public.go b/server/public.go index acafd08c..abe67790 100644 --- a/server/public.go +++ b/server/public.go @@ -18,8 +18,10 @@ import ( ) const blockbookAbout = "Blockbook - blockchain indexer for TREZOR wallet https://trezor.io/. Do not use for any other purpose." +const txsOnPage = 30 +const txsInAPI = 1000 -// PublicServer is handle to public http server +// PublicServer is a handle to public http server type PublicServer struct { binding string certFiles string @@ -37,7 +39,7 @@ type PublicServer struct { addressTpl *template.Template } -// NewPublicServerS creates new public server http interface to blockbook and returns its handle +// NewPublicServer creates new public server http interface to blockbook and returns its handle func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState) (*PublicServer, error) { api, err := api.NewWorker(db, chain, txCache, is) @@ -233,9 +235,10 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) { page = 0 } addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page) + address, err = s.api.GetAddress(addrID, page, txsOnPage) if err != nil { glog.Error(err) + // TODO return error.html } } w.Header().Set("Content-Type", "text/html; charset=utf-8") @@ -351,7 +354,7 @@ func (s *PublicServer) apiAddress(w http.ResponseWriter, r *http.Request) { page = 0 } addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page) + address, err = s.api.GetAddress(addrID, page, txsInAPI) if err != nil { glog.Error(err) } From 78f6162d5c9e2f6c5ae5c6c86bf3bcd0d40aa8c7 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 21 Aug 2018 13:16:29 +0200 Subject: [PATCH 022/123] Add height to TxAddresses as required by explorer --- db/rocksdb.go | 20 ++++++++++++++------ db/rocksdb_test.go | 47 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index ef3be7db..7a34e47a 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -535,6 +535,7 @@ func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, error) { } type TxAddresses struct { + Height uint32 Inputs []TxInput Outputs []TxOutput } @@ -569,6 +570,7 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { blockTxIDs := make([][]byte, len(block.Txs)) + blockTxAddresses := make([]*TxAddresses, len(block.Txs)) // first process all outputs so that inputs can point to txs in this block for txi := range block.Txs { tx := &block.Txs[txi] @@ -577,9 +579,10 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string return err } blockTxIDs[txi] = btxID - ta := TxAddresses{} + ta := TxAddresses{Height: block.Height} ta.Outputs = make([]TxOutput, len(tx.Vout)) txAddressesMap[string(btxID)] = &ta + blockTxAddresses[txi] = &ta for i, output := range tx.Vout { tao := &ta.Outputs[i] tao.ValueSat = output.ValueSat @@ -629,7 +632,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string for txi := range block.Txs { tx := &block.Txs[txi] spendingTxid := blockTxIDs[txi] - ta := txAddressesMap[string(spendingTxid)] + ta := blockTxAddresses[txi] ta.Inputs = make([]TxInput, len(tx.Vin)) for i, input := range tx.Vin { tai := &ta.Inputs[i] @@ -874,8 +877,8 @@ func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { } defer val.Free() buf := val.Data() - // 2 is minimum length of addrBalance - 1 byte inputs len, 1 byte outputs len - if len(buf) < 2 { + // 2 is minimum length of addrBalance - 1 byte height, 1 byte inputs len, 1 byte outputs len + if len(buf) < 3 { return nil, nil } return unpackTxAddresses(buf) @@ -892,7 +895,9 @@ func (d *RocksDB) GetTxAddresses(txid string) (*TxAddresses, error) { func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { buf = buf[:0] - l := packVaruint(uint(len(ta.Inputs)), varBuf) + l := packVaruint(uint(ta.Height), varBuf) + buf = append(buf, varBuf[:l]...) + l = packVaruint(uint(len(ta.Inputs)), varBuf) buf = append(buf, varBuf[:l]...) for i := range ta.Inputs { buf = appendTxInput(&ta.Inputs[i], buf, varBuf) @@ -930,7 +935,10 @@ func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { func unpackTxAddresses(buf []byte) (*TxAddresses, error) { ta := TxAddresses{} - inputs, l := unpackVaruint(buf) + height, l := unpackVaruint(buf) + ta.Height = uint32(height) + inputs, ll := unpackVaruint(buf[l:]) + l += ll ta.Inputs = make([]TxInput, inputs) for i := uint(0); i < inputs; i++ { l += unpackTxInput(&ta.Inputs[i], buf[l:]) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index cd33f6ed..7c61a5a1 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -16,6 +16,7 @@ import ( "strings" "testing" + vlq "github.com/bsm/go-vlq" "github.com/juju/errors" ) @@ -85,6 +86,12 @@ func bigintToHex(i *big.Int) string { return hex.EncodeToString(b[:l]) } +func varuintToHex(i uint) string { + b := make([]byte, vlq.MaxLen64) + l := vlq.PutUint(b, uint64(i)) + return hex.EncodeToString(b[:l]) +} + // keyPair is used to compare given key value in DB with expected // for more complicated compares it is possible to specify CompareFunc type keyPair struct { @@ -382,14 +389,18 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ txidB1T1, - "00" + "02" + + varuintToHex(225493) + + "00" + + "02" + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), nil, }, keyPair{ txidB1T2, - "00" + "03" + + varuintToHex(225493) + + "00" + + "03" + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), @@ -464,14 +475,18 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ txidB1T1, - "00" + "02" + + varuintToHex(225493) + + "00" + + "02" + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + spentAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), nil, }, keyPair{ txidB1T2, - "00" + "03" + + varuintToHex(225493) + + "00" + + "03" + spentAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + spentAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + spentAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), @@ -479,7 +494,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { }, keyPair{ txidB2T1, - "02" + + varuintToHex(225494) + + "02" + inputAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + inputAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "02" + @@ -489,7 +505,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { }, keyPair{ txidB2T2, - "02" + + varuintToHex(225494) + + "02" + inputAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + inputAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "02" + @@ -499,7 +516,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { }, keyPair{ txidB2T3, - "01" + + varuintToHex(225494) + + "01" + inputAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "01" + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), @@ -507,7 +525,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { }, keyPair{ txidB2T4, - "01" + inputAddressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero) + + varuintToHex(225494) + + "01" + inputAddressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero) + "02" + addressToPubKeyHexWithLength(addrA, t, d) + bigintToHex(satB2T4AA) + addressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero), @@ -747,6 +766,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } taw := &TxAddresses{ + Height: 225494, Inputs: []TxInput{ { addrID: addressToOutput(addr3, d.chainParser), @@ -929,8 +949,9 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }{ { name: "1", - hex: "0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + hex: "7b0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", data: &TxAddresses{ + Height: 123, Inputs: []TxInput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), @@ -952,8 +973,9 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "2", - hex: "0317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + hex: "e0390317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", data: &TxAddresses{ + Height: 12345, Inputs: []TxInput{ { addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), @@ -997,8 +1019,9 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "empty address", - hex: "01000204d2020002162e010162", + hex: "baef9a1501000204d2020002162e010162", data: &TxAddresses{ + Height: 123456789, Inputs: []TxInput{ { addrID: []byte{}, @@ -1020,7 +1043,7 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "empty", - hex: "0000", + hex: "000000", data: &TxAddresses{ Inputs: []TxInput{}, Outputs: []TxOutput{}, From c9471bf867fcd59f1b11348dbf161c5ddd1356b5 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 21 Aug 2018 16:36:14 +0200 Subject: [PATCH 023/123] Store extended info about block in heigth column --- bchain/coins/btc/bitcoinparser.go | 8 +++- bchain/coins/eth/ethrpc.go | 4 +- bchain/types.go | 2 + db/rocksdb.go | 73 ++++++++++++++++++++++++++++--- db/rocksdb_test.go | 54 +++++++++++++++++++++-- 5 files changed, 130 insertions(+), 11 deletions(-) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 767e8d08..6e2f4c69 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -184,7 +184,13 @@ func (p *BitcoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: w.Header.Timestamp.Unix(), + }, + Txs: txs, + }, nil } // PackTx packs transaction to byte array diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 0a19e10b..2ef90b5d 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -321,9 +321,9 @@ func (b *EthereumRPC) ethHeaderToBlockHeader(h *ethtypes.Header) (*bchain.BlockH Hash: ethHashToHash(h.Hash()), Height: uint32(hn), Confirmations: int(c), + Time: int64(h.Time.Uint64()), // Next // Prev - }, nil } @@ -393,6 +393,8 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error return nil, errors.Annotatef(fmt.Errorf("server returned empty transaction list but block header indicates transactions"), "hash %v, height %v", hash, height) } bbh, err := b.ethHeaderToBlockHeader(head) + // TODO - this is probably not the correct size + bbh.Size = len(raw) btxs := make([]bchain.Tx, len(body.Transactions)) for i, tx := range body.Transactions { btx, err := b.Parser.ethTxToTx(&tx, int64(head.Time.Uint64()), uint32(bbh.Confirmations)) diff --git a/bchain/types.go b/bchain/types.go index cfe90771..ba631310 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -88,6 +88,8 @@ type BlockHeader struct { Next string `json:"nextblockhash"` Height uint32 `json:"height"` Confirmations int `json:"confirmations"` + Size int `json:"size"` + Time int64 `json:"time,omitempty"` } type MempoolEntry struct { diff --git a/db/rocksdb.go b/db/rocksdb.go index 7a34e47a..1ecc4f61 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -1091,17 +1091,63 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // Block index +type BlockInfo struct { + Txid string + Time time.Time + Txs uint32 + Size uint32 +} + +func (d *RocksDB) packBlockInfo(block *bchain.Block) ([]byte, error) { + packed := make([]byte, 0, 64) + varBuf := make([]byte, vlq.MaxLen64) + b, err := d.chainParser.PackBlockHash(block.Hash) + if err != nil { + return nil, err + } + packed = append(packed, b...) + packed = append(packed, packUint(uint32(block.Time))...) + l := packVaruint(uint(len(block.Txs)), varBuf) + packed = append(packed, varBuf[:l]...) + l = packVaruint(uint(block.Size), varBuf) + packed = append(packed, varBuf[:l]...) + return packed, nil +} + +func (d *RocksDB) unpackBlockInfo(buf []byte) (*BlockInfo, error) { + pl := d.chainParser.PackedTxidLen() + // minimum length is PackedTxidLen+4 bytes time + 1 byte txs + 1 byte size + if len(buf) < pl+4+2 { + return nil, nil + } + txid, err := d.chainParser.UnpackBlockHash(buf[:pl]) + if err != nil { + return nil, err + } + t := unpackUint(buf[pl:]) + txs, l := unpackVaruint(buf[pl+4:]) + size, _ := unpackVaruint(buf[pl+4+l:]) + return &BlockInfo{ + Txid: txid, + Time: time.Unix(int64(t), 0), + Txs: uint32(txs), + Size: uint32(size), + }, nil +} + // GetBestBlock returns the block hash of the block with highest height in the db func (d *RocksDB) GetBestBlock() (uint32, string, error) { it := d.db.NewIteratorCF(d.ro, d.cfh[cfHeight]) defer it.Close() if it.SeekToLast(); it.Valid() { bestHeight := unpackUint(it.Key().Data()) - val, err := d.chainParser.UnpackBlockHash(it.Value().Data()) - if glog.V(1) { - glog.Infof("rocksdb: bestblock %d %s", bestHeight, val) + info, err := d.unpackBlockInfo(it.Value().Data()) + if info != nil { + if glog.V(1) { + glog.Infof("rocksdb: bestblock %d %+v", bestHeight, info) + } + return bestHeight, info.Txid, err } - return bestHeight, val, err } return 0, "", nil } @@ -1114,7 +1160,22 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { return "", err } defer val.Free() - return d.chainParser.UnpackBlockHash(val.Data()) + info, err := d.unpackBlockInfo(val.Data()) + if info == nil { + return "", err + } + return info.Txid, nil +} + +// GetBlockInfo returns block info stored in db +func (d *RocksDB) GetBlockInfo(height uint32) (*BlockInfo, error) { + key := packUint(height) + val, err := d.db.GetCF(d.ro, d.cfh[cfHeight], key) + if err != nil { + return nil, err + } + defer val.Free() + return d.unpackBlockInfo(val.Data()) } func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { @@ -1122,7 +1183,7 @@ func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op switch op { case opInsert: - val, err := d.chainParser.PackBlockHash(block.Hash) + val, err := d.packBlockInfo(block) if err != nil { return err } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 7c61a5a1..732ae410 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "blockbook/common" + "encoding/binary" "encoding/hex" "fmt" "io/ioutil" @@ -15,6 +16,7 @@ import ( "sort" "strings" "testing" + "time" vlq "github.com/bsm/go-vlq" "github.com/juju/errors" @@ -92,6 +94,12 @@ func varuintToHex(i uint) string { return hex.EncodeToString(b[:l]) } +func uintToHex(i uint32) string { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, i) + return hex.EncodeToString(buf) +} + // keyPair is used to compare given key value in DB with expected // for more complicated compares it is possible to specify CompareFunc type keyPair struct { @@ -187,6 +195,8 @@ func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { BlockHeader: bchain.BlockHeader{ Height: 225493, Hash: "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", + Size: 1234567, + Time: 1534858021, }, Txs: []bchain.Tx{ bchain.Tx{ @@ -247,6 +257,8 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { BlockHeader: bchain.BlockHeader{ Height: 225494, Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Size: 2345678, + Time: 1534859123, }, Txs: []bchain.Tx{ bchain.Tx{ @@ -368,7 +380,11 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, + keyPair{ + "000370d5", + "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997" + uintToHex(1534858021) + varuintToHex(2) + varuintToHex(1234567), + nil, + }, }); err != nil { { t.Fatal(err) @@ -445,8 +461,16 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, - keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", nil}, + keyPair{ + "000370d5", + "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997" + uintToHex(1534858021) + varuintToHex(2) + varuintToHex(1234567), + nil, + }, + keyPair{ + "000370d6", + "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859123) + varuintToHex(4) + varuintToHex(2345678), + nil, + }, }); err != nil { { t.Fatal(err) @@ -692,6 +716,30 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatalf("GetBlockHash: got hash %v, expected %v", hash, "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997") } + // Not connected block + hash, err = d.GetBlockHash(225495) + if err != nil { + t.Fatal(err) + } + if hash != "" { + t.Fatalf("GetBlockHash: got hash '%v', expected ''", hash) + } + + // GetBlockHash + info, err := d.GetBlockInfo(225494) + if err != nil { + t.Fatal(err) + } + iw := &BlockInfo{ + Txid: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Txs: 4, + Size: 2345678, + Time: time.Unix(1534859123, 0), + } + if !reflect.DeepEqual(info, iw) { + t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) + } + // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock testTxCache(t, d, block1, &block1.Txs[0]) testTxCache(t, d, block2, &block2.Txs[0]) From 188eed8881877ada24c3a14f8ec4f913fd0d6d88 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 21 Aug 2018 18:56:30 +0200 Subject: [PATCH 024/123] Write data to DB synchronously in bulk connect RocksDB has some memory leak/fragmentation problem when inserting data in parallel --- db/rocksdb.go | 145 +++++++++++++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 56 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 1ecc4f61..6c4ebf9f 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -285,12 +285,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return d.db.Write(d.wo, wb) } -// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +// bulk connect +// in bulk mode the data are cached and stored to db in batches +// it speeds up the import in two ways: +// 1) balances and txAddresses are modified several times during the import, there is a chance that the modifications are done before write to DB +// 2) rocksdb seems to handle better fewer larger batches than continuous stream of smaller batches + type bulkAddresses struct { height uint32 addresses map[string][]outpoint } +// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way type BulkConnect struct { d *RocksDB isUTXO bool @@ -302,13 +308,14 @@ type BulkConnect struct { } const ( - maxBulkAddresses = 400000 + maxBulkAddresses = 300000 maxBulkTxAddresses = 2000000 partialStoreAddresses = maxBulkTxAddresses / 10 maxBulkBalances = 2500000 partialStoreBalances = maxBulkBalances / 10 ) +// InitBulkConnect initializes bulk connect and switches DB to inconsistent state func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { bc := &BulkConnect{ d: d, @@ -323,9 +330,7 @@ func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { return bc, nil } -func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { - defer close(c) - start := time.Now() +func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int, int, error) { var txm map[string]*TxAddresses var sp int if all { @@ -359,21 +364,31 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { } } } - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() if err := b.d.storeTxAddresses(wb, txm); err != nil { - c <- err - } else { - if err := b.d.db.Write(b.d.wo, wb); err != nil { - c <- err - } + return 0, 0, err } - glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + return len(txm), sp, nil } -func (b *BulkConnect) storeBalances(c chan error, all bool) { +func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) { defer close(c) start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + count, sp, err := b.storeTxAddresses(wb, all) + if err != nil { + c <- err + return + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + return + } + glog.Info("rocksdb: height ", b.height, ", stored ", count, " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + c <- nil +} + +func (b *BulkConnect) storeBalances(wb *gorocksdb.WriteBatch, all bool) (int, error) { var bal map[string]*AddrBalance if all { bal = b.balances @@ -389,16 +404,28 @@ func (b *BulkConnect) storeBalances(c chan error, all bool) { } } } + if err := b.d.storeBalances(wb, bal); err != nil { + return 0, err + } + return len(bal), nil +} + +func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) { + defer close(c) + start := time.Now() wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - if err := b.d.storeBalances(wb, bal); err != nil { + count, err := b.storeBalances(wb, all) + if err != nil { c <- err - } else { - if err := b.d.db.Write(b.d.wo, wb); err != nil { - c <- err - } + return } - glog.Info("rocksdb: height ", b.height, ", stored ", len(bal), " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + return + } + glog.Info("rocksdb: height ", b.height, ", stored ", count, " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) + c <- nil } func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { @@ -412,41 +439,46 @@ func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { return nil } +// ConnectBlock connects block in bulk mode func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { b.height = block.Height if !b.isUTXO { return b.d.ConnectBlock(block) } - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() addresses := make(map[string][]outpoint) if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { return err } - start := time.Now() - var sa bool - var storeAddressesChan, storeBalancesChan chan error - if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { - sa = true - if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { - storeAddressesChan = make(chan error) - go b.storeTxAddresses(storeAddressesChan, false) - } - if len(b.balances)+partialStoreBalances > maxBulkBalances { - storeBalancesChan = make(chan error) - go b.storeBalances(storeBalancesChan, false) - } - } b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ height: block.Height, addresses: addresses, }) b.bulkAddressesCount += len(addresses) - bac := b.bulkAddressesCount - if sa || b.bulkAddressesCount > maxBulkAddresses { - if err := b.storeBulkAddresses(wb); err != nil { - return err - } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + start := time.Now() + var count, count1, storeType int + var err error + const ( + storeNone = iota + storeTxAddresses + storeBalances + storeBulkAddresses + ) + // in one block store maximally one type of data + if len(b.txAddressesMap) > maxBulkTxAddresses { + storeType = storeTxAddresses + count, count1, err = b.storeTxAddresses(wb, false) + } else if len(b.balances) > maxBulkBalances { + storeType = storeBalances + count, err = b.storeBalances(wb, false) + } else if b.bulkAddressesCount > maxBulkAddresses { + storeType = storeBulkAddresses + count = b.bulkAddressesCount + err = b.storeBulkAddresses(wb) + } + if err != nil { + return err } if storeBlockTxs { if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { @@ -459,29 +491,26 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro if err := b.d.db.Write(b.d.wo, wb); err != nil { return err } - if bac > b.bulkAddressesCount { - glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) - } - if storeAddressesChan != nil { - if err := <-storeAddressesChan; err != nil { - return err - } - } - if storeBalancesChan != nil { - if err := <-storeBalancesChan; err != nil { - return err - } + switch storeType { + case storeTxAddresses: + glog.Info("rocksdb: height ", b.height, ", stored ", count, " (", count1, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + case storeBalances: + glog.Info("rocksdb: height ", b.height, ", stored ", count, " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) + case storeBulkAddresses: + glog.Info("rocksdb: height ", b.height, ", stored ", count, " addresses, done in ", time.Since(start)) } return nil } +// Close flushes the cached data and switches DB from inconsistent state open +// after Close, the BulkConnect cannot be used func (b *BulkConnect) Close() error { glog.Info("rocksdb: bulk connect closing") start := time.Now() storeAddressesChan := make(chan error) - go b.storeTxAddresses(storeAddressesChan, true) + go b.parallelStoreTxAddresses(storeAddressesChan, true) storeBalancesChan := make(chan error) - go b.storeBalances(storeBalancesChan, true) + go b.parallelStoreBalances(storeBalancesChan, true) wb := gorocksdb.NewWriteBatch() defer wb.Destroy() bac := b.bulkAddressesCount @@ -634,6 +663,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string spendingTxid := blockTxIDs[txi] ta := blockTxAddresses[txi] ta.Inputs = make([]TxInput, len(tx.Vin)) + logged := false for i, input := range tx.Vin { tai := &ta.Inputs[i] btxID, err := d.chainParser.PackTxid(input.Txid) @@ -670,7 +700,10 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string // mark the output as spent in tx ot.Spent = true if len(ot.addrID) == 0 { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) + if !logged { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) + logged = true + } continue } strAddrID := string(ot.addrID) From 59497e3c975dbc72314ed4472a8ae62ddea90058 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 22 Aug 2018 00:48:53 +0200 Subject: [PATCH 025/123] Get address data for explorer using index v3 - WIP --- api/worker.go | 84 ++++++++++++++++++++++++++++++++++++++++++---- db/rocksdb.go | 20 +++++------ db/rocksdb_test.go | 8 ++--- 3 files changed, 91 insertions(+), 21 deletions(-) diff --git a/api/worker.go b/api/worker.go index 64a93bff..b0ddfe89 100644 --- a/api/worker.go +++ b/api/worker.go @@ -170,8 +170,57 @@ func UniqueTxidsInReverse(txids []string) []string { return ut[i:] } +func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx { + var valInSat, valOutSat, feesSat big.Int + vins := make([]Vin, len(ta.Inputs)) + for i := range ta.Inputs { + tai := &ta.Inputs[i] + vin := &vins[i] + vin.N = i + vin.ValueSat = tai.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + valInSat.Add(&valInSat, &vin.ValueSat) + a, err := tai.Addresses(w.chainParser) + if err != nil && len(a) == 1 { + vin.Addr = a[0] + } + } + vouts := make([]Vout, len(ta.Outputs)) + for i := range ta.Outputs { + tao := &ta.Outputs[i] + vout := &vouts[i] + vout.N = i + vout.ValueSat = tao.ValueSat + vout.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) + valOutSat.Add(&valOutSat, &vout.ValueSat) + a, err := tao.Addresses(w.chainParser) + if err != nil { + vout.ScriptPubKey.Addresses = a + } + } + // for coinbase transactions valIn is 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) + } + r := &Tx{ + Blockhash: bi.BlockHash, + Blockheight: int(ta.Height), + Blocktime: bi.Time.Unix(), + Confirmations: bestheight - ta.Height + 1, + Fees: w.chainParser.AmountToDecimalString(&feesSat), + Time: bi.Time.Unix(), + Txid: txid, + ValueIn: w.chainParser.AmountToDecimalString(&valInSat), + ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), + Vin: vins, + Vout: vouts, + } + return r +} + // GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddressNew(address string, page int, txsOnPage int) (*Address, error) { +func (w *Worker) GetAddress(address string, page int, txsOnPage int) (*Address, error) { glog.Info(address, " start") ba, err := w.db.GetAddressBalance(address) if err != nil { @@ -227,9 +276,29 @@ func (w *Worker) GetAddressNew(address string, page int, txsOnPage int) (*Addres } } if len(txc) != int(ba.Txs) { - glog.Warning("DB inconsistency in address ", address, ": number of txs from column addresses ", len(txc), ", from addressBalance ", ba.Txs) + glog.Warning("DB inconsistency for address ", address, ": number of txs from column addresses ", len(txc), ", from addressBalance ", ba.Txs) + } + for i := from; i < to; i++ { + txid := txc[i] + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, err + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + bi, err := w.db.GetBlockInfo(ta.Height) + if err != nil { + return nil, err + } + if bi == nil { + glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) + txi++ } - r := &Address{ AddrStr: address, Balance: w.chainParser.AmountToDecimalString(&ba.BalanceSat), @@ -238,16 +307,17 @@ func (w *Worker) GetAddressNew(address string, page int, txsOnPage int) (*Addres TxApperances: len(txc), UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), UnconfirmedTxApperances: len(txm), - Page: page, - TotalPages: totalPages, - TxsOnPage: txsOnPage, + Transactions: txs[:txi], + Page: page, + TotalPages: totalPages, + TxsOnPage: txsOnPage, } glog.Info(address, " finished") return r, nil } // GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddress(addrID string, page int, txsOnPage int) (*Address, error) { +func (w *Worker) GetAddressOld(addrID string, page int, txsOnPage int) (*Address, error) { glog.Info(addrID, " start") txc, err := w.getAddressTxids(addrID, false) txc = UniqueTxidsInReverse(txc) diff --git a/db/rocksdb.go b/db/rocksdb.go index 6c4ebf9f..6f6b3793 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -1125,10 +1125,10 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // Block index type BlockInfo struct { - Txid string - Time time.Time - Txs uint32 - Size uint32 + BlockHash string + Time time.Time + Txs uint32 + Size uint32 } func (d *RocksDB) packBlockInfo(block *bchain.Block) ([]byte, error) { @@ -1161,10 +1161,10 @@ func (d *RocksDB) unpackBlockInfo(buf []byte) (*BlockInfo, error) { txs, l := unpackVaruint(buf[pl+4:]) size, _ := unpackVaruint(buf[pl+4+l:]) return &BlockInfo{ - Txid: txid, - Time: time.Unix(int64(t), 0), - Txs: uint32(txs), - Size: uint32(size), + BlockHash: txid, + Time: time.Unix(int64(t), 0), + Txs: uint32(txs), + Size: uint32(size), }, nil } @@ -1179,7 +1179,7 @@ func (d *RocksDB) GetBestBlock() (uint32, string, error) { if glog.V(1) { glog.Infof("rocksdb: bestblock %d %+v", bestHeight, info) } - return bestHeight, info.Txid, err + return bestHeight, info.BlockHash, err } } return 0, "", nil @@ -1197,7 +1197,7 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { if info == nil { return "", err } - return info.Txid, nil + return info.BlockHash, nil } // GetBlockInfo returns block info stored in db diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 732ae410..e74d0664 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -731,10 +731,10 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } iw := &BlockInfo{ - Txid: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", - Txs: 4, - Size: 2345678, - Time: time.Unix(1534859123, 0), + BlockHash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Txs: 4, + Size: 2345678, + Time: time.Unix(1534859123, 0), } if !reflect.DeepEqual(info, iw) { t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) From 41252d33d23adfaede2247276c10686c96c81f16 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 22 Aug 2018 16:20:52 +0200 Subject: [PATCH 026/123] Tune RocksDB options and measure memory usage --- db/rocksdb.go | 176 +++++++++++++++++++++++++++++--------------------- db/sync.go | 5 ++ 2 files changed, 106 insertions(+), 75 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 6f6b3793..35a52cde 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -6,6 +6,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "fmt" "math/big" "os" "path/filepath" @@ -42,6 +43,7 @@ type RocksDB struct { chainParser bchain.BlockChainParser is *common.InternalState metrics *common.Metrics + cache *gorocksdb.Cache } const ( @@ -56,57 +58,49 @@ const ( var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} -func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { - c := gorocksdb.NewLRUCache(8 << 30) // 8GB - fp := gorocksdb.NewBloomFilter(10) - bbto := gorocksdb.NewDefaultBlockBasedTableOptions() - bbto.SetBlockSize(16 << 10) // 16kB - bbto.SetBlockCache(c) - bbto.SetFilterPolicy(fp) +func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { + bloom := gorocksdb.NewBloomFilter(10) + blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() + blockOpts.SetBlockSize(16 << 10) // 16kB + blockOpts.SetBlockCache(c) + blockOpts.SetFilterPolicy(bloom) + blockOpts.SetCacheIndexAndFilterBlocks(true) + blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) - optsNoCompression := gorocksdb.NewDefaultOptions() - optsNoCompression.SetBlockBasedTableFactory(bbto) - optsNoCompression.SetCreateIfMissing(true) - optsNoCompression.SetCreateIfMissingColumnFamilies(true) - optsNoCompression.SetMaxBackgroundCompactions(4) - optsNoCompression.SetMaxBackgroundFlushes(2) - optsNoCompression.SetBytesPerSync(1 << 20) // 1MB - optsNoCompression.SetWriteBufferSize(1 << 27) // 128MB - optsNoCompression.SetMaxOpenFiles(25000) - optsNoCompression.SetCompression(gorocksdb.NoCompression) - - optsLZ4 := gorocksdb.NewDefaultOptions() - optsLZ4.SetBlockBasedTableFactory(bbto) - optsLZ4.SetCreateIfMissing(true) - optsLZ4.SetCreateIfMissingColumnFamilies(true) - optsLZ4.SetMaxBackgroundCompactions(4) - optsLZ4.SetMaxBackgroundFlushes(2) - optsLZ4.SetBytesPerSync(1 << 20) // 1MB - optsLZ4.SetWriteBufferSize(1 << 27) // 128MB - optsLZ4.SetMaxOpenFiles(25000) - optsLZ4.SetCompression(gorocksdb.LZ4HCCompression) + opts := gorocksdb.NewDefaultOptions() + opts.SetBlockBasedTableFactory(blockOpts) + opts.SetCreateIfMissing(true) + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetMaxBackgroundCompactions(6) + opts.SetMaxBackgroundFlushes(6) + opts.SetBytesPerSync(1 << 20) // 1MB + opts.SetWriteBufferSize(1 << 27) // 128MB + opts.SetMaxOpenFiles(25000) + opts.SetCompression(gorocksdb.LZ4HCCompression) // opts for addresses are different: // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter - bbtoAddresses := gorocksdb.NewDefaultBlockBasedTableOptions() - bbtoAddresses.SetBlockSize(16 << 10) // 16kB - bbtoAddresses.SetBlockCache(c) // 8GB + blockOptsAddress := gorocksdb.NewDefaultBlockBasedTableOptions() + blockOptsAddress.SetBlockSize(16 << 10) // 16kB + blockOptsAddress.SetBlockCache(c) // 8GB + blockOptsAddress.SetCacheIndexAndFilterBlocks(true) + blockOptsAddress.SetPinL0FilterAndIndexBlocksInCache(true) optsAddresses := gorocksdb.NewDefaultOptions() - optsAddresses.SetBlockBasedTableFactory(bbtoAddresses) + optsAddresses.SetBlockBasedTableFactory(blockOptsAddress) optsAddresses.SetCreateIfMissing(true) optsAddresses.SetCreateIfMissingColumnFamilies(true) - optsAddresses.SetMaxBackgroundCompactions(4) - optsAddresses.SetMaxBackgroundFlushes(2) + optsAddresses.SetMaxBackgroundCompactions(6) + optsAddresses.SetMaxBackgroundFlushes(6) optsAddresses.SetBytesPerSync(1 << 20) // 1MB optsAddresses.SetWriteBufferSize(1 << 27) // 128MB optsAddresses.SetMaxOpenFiles(25000) optsAddresses.SetCompression(gorocksdb.LZ4HCCompression) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions - fcOptions := []*gorocksdb.Options{optsLZ4, optsLZ4, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} + fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions) + db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) if err != nil { return nil, nil, err } @@ -117,11 +111,11 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) // needs to be called to release it. func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { glog.Infof("rocksdb: open %s, version %v", path, dbVersion) - db, cfh, err := openDB(path) + c := gorocksdb.NewLRUCache(8 << 30) // 8GB + db, cfh, err := openDB(path, c) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() - ro.SetFillCache(false) - return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics}, nil + return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c}, nil } func (d *RocksDB) closeDB() error { @@ -159,7 +153,7 @@ func (d *RocksDB) Reopen() error { return err } d.db = nil - db, cfh, err := openDB(d.path) + db, cfh, err := openDB(d.path, d.cache) if err != nil { return err } @@ -167,6 +161,34 @@ func (d *RocksDB) Reopen() error { return nil } +func (d *RocksDB) GetMemoryStats() string { + type columnStats struct { + name string + indexAndFilter string + memtable string + } + cs := make([]columnStats, len(cfNames)) + for i := 0; i < len(cfNames); i++ { + cs[i].name = cfNames[i] + cs[i].indexAndFilter = d.db.GetPropertyCF("rocksdb.estimate-table-readers-mem", d.cfh[i]) + cs[i].memtable = d.db.GetPropertyCF("rocksdb.cur-size-all-mem-tables", d.cfh[i]) + } + m := struct { + cacheUsage int + pinnedCacheUsage int + indexAndFilter string + memtable string + columns []columnStats + }{ + cacheUsage: d.cache.GetUsage(), + pinnedCacheUsage: d.cache.GetPinnedUsage(), + indexAndFilter: d.db.GetProperty("rocksdb.estimate-table-readers-mem"), + memtable: d.db.GetProperty("rocksdb.cur-size-all-mem-tables"), + columns: cs, + } + return fmt.Sprintf("%+v", m) +} + // GetTransactions finds all input/output transactions for address // Transaction are passed to callback function. func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { @@ -308,8 +330,8 @@ type BulkConnect struct { } const ( - maxBulkAddresses = 300000 - maxBulkTxAddresses = 2000000 + maxBulkAddresses = 400000 + maxBulkTxAddresses = 1500000 partialStoreAddresses = maxBulkTxAddresses / 10 maxBulkBalances = 2500000 partialStoreBalances = maxBulkBalances / 10 @@ -445,40 +467,36 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro if !b.isUTXO { return b.d.ConnectBlock(block) } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() addresses := make(map[string][]outpoint) if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { return err } + start := time.Now() + var sa bool + var storeAddressesChan, storeBalancesChan chan error + if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + sa = true + if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { + storeAddressesChan = make(chan error) + go b.parallelStoreTxAddresses(storeAddressesChan, false) + } + if len(b.balances)+partialStoreBalances > maxBulkBalances { + storeBalancesChan = make(chan error) + go b.parallelStoreBalances(storeBalancesChan, false) + } + } b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ height: block.Height, addresses: addresses, }) b.bulkAddressesCount += len(addresses) - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - start := time.Now() - var count, count1, storeType int - var err error - const ( - storeNone = iota - storeTxAddresses - storeBalances - storeBulkAddresses - ) - // in one block store maximally one type of data - if len(b.txAddressesMap) > maxBulkTxAddresses { - storeType = storeTxAddresses - count, count1, err = b.storeTxAddresses(wb, false) - } else if len(b.balances) > maxBulkBalances { - storeType = storeBalances - count, err = b.storeBalances(wb, false) - } else if b.bulkAddressesCount > maxBulkAddresses { - storeType = storeBulkAddresses - count = b.bulkAddressesCount - err = b.storeBulkAddresses(wb) - } - if err != nil { - return err + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } } if storeBlockTxs { if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { @@ -491,13 +509,18 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro if err := b.d.db.Write(b.d.wo, wb); err != nil { return err } - switch storeType { - case storeTxAddresses: - glog.Info("rocksdb: height ", b.height, ", stored ", count, " (", count1, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) - case storeBalances: - glog.Info("rocksdb: height ", b.height, ", stored ", count, " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) - case storeBulkAddresses: - glog.Info("rocksdb: height ", b.height, ", stored ", count, " addresses, done in ", time.Since(start)) + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + } + if storeAddressesChan != nil { + if err := <-storeAddressesChan; err != nil { + return err + } + } + if storeBalancesChan != nil { + if err := <-storeBalancesChan; err != nil { + return err + } } return nil } @@ -1628,9 +1651,12 @@ func (d *RocksDB) storeState(is *common.InternalState) error { func (d *RocksDB) computeColumnSize(col int, stopCompute chan os.Signal) (int64, int64, int64, error) { var rows, keysSum, valuesSum int64 var seekKey []byte + // do not use cache + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) for { var key []byte - it := d.db.NewIteratorCF(d.ro, d.cfh[col]) + it := d.db.NewIteratorCF(ro, d.cfh[col]) if rows == 0 { it.SeekToFirst() } else { diff --git a/db/sync.go b/db/sync.go index 32bc143d..8f52e589 100644 --- a/db/sync.go +++ b/db/sync.go @@ -286,6 +286,7 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { go writeBlockWorker() var hash string start := time.Now() + msTime := time.Now().Add(1 * time.Minute) ConnectLoop: for h := lower; h <= higher; { select { @@ -305,6 +306,10 @@ ConnectLoop: glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start)) start = time.Now() } + if msTime.Before(time.Now()) { + glog.Info(w.db.GetMemoryStats()) + msTime = time.Now().Add(1 * time.Minute) + } h++ } } From a34ac148834581dcae83ae1354afda79e3ad03c3 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 23 Aug 2018 11:15:59 +0200 Subject: [PATCH 027/123] Allow partitioned index filter in rocksdb --- db/dboptions.go | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ db/rocksdb.go | 46 +++++----------------------------- 2 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 db/dboptions.go diff --git a/db/dboptions.go b/db/dboptions.go new file mode 100644 index 00000000..81ff82d3 --- /dev/null +++ b/db/dboptions.go @@ -0,0 +1,65 @@ +package db + +// #include "rocksdb/c.h" +import "C" +import ( + "reflect" + "unsafe" + + "github.com/tecbot/gorocksdb" +) + +func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache) *gorocksdb.Options { + // get access to all db options, gorocksdb does not expose all + + // opts := gorocksdb.NewDefaultOptions() + cNativeOpts := C.rocksdb_options_create() + opts := &gorocksdb.Options{} + cField := reflect.Indirect(reflect.ValueOf(opts)).FieldByName("c") + cPtr := (**C.rocksdb_options_t)(unsafe.Pointer(cField.UnsafeAddr())) + *cPtr = cNativeOpts + + // blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() + cNativeBlockOpts := C.rocksdb_block_based_options_create() + blockOpts := &gorocksdb.BlockBasedTableOptions{} + cBlockField := reflect.Indirect(reflect.ValueOf(blockOpts)).FieldByName("c") + cBlockPtr := (**C.rocksdb_block_based_table_options_t)(unsafe.Pointer(cBlockField.UnsafeAddr())) + *cBlockPtr = cNativeBlockOpts + + blockOpts.SetBlockSize(32 << 10) // 32kB + blockOpts.SetBlockCache(c) + if bloomBits > 0 { + blockOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(bloomBits)) + } + + // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters + if twoLevelIndex { + // blockOpts.SetCacheIndexAndFilterBlocks(true) + blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) + C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true)) + C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096)) + C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true)) + blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) + } + + opts.SetBlockBasedTableFactory(blockOpts) + opts.SetCreateIfMissing(true) + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetMaxBackgroundCompactions(6) + opts.SetMaxBackgroundFlushes(6) + opts.SetBytesPerSync(8 << 20) // 8MB + opts.SetWriteBufferSize(1 << 27) // 128MB + opts.SetMaxBytesForLevelBase(1 << 27) // 128MB + opts.SetMaxOpenFiles(25000) + opts.SetCompression(gorocksdb.LZ4HCCompression) + + return opts +} + +// boolToChar converts a bool value to C.uchar. +func boolToChar(b bool) C.uchar { + if b { + return 1 + } + return 0 +} diff --git a/db/rocksdb.go b/db/rocksdb.go index 35a52cde..704f3a90 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -59,47 +59,13 @@ const ( var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { - bloom := gorocksdb.NewBloomFilter(10) - blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() - blockOpts.SetBlockSize(16 << 10) // 16kB - blockOpts.SetBlockCache(c) - blockOpts.SetFilterPolicy(bloom) - blockOpts.SetCacheIndexAndFilterBlocks(true) - blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) - - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(blockOpts) - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) - opts.SetMaxBackgroundCompactions(6) - opts.SetMaxBackgroundFlushes(6) - opts.SetBytesPerSync(1 << 20) // 1MB - opts.SetWriteBufferSize(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) - opts.SetCompression(gorocksdb.LZ4HCCompression) - - // opts for addresses are different: - // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter - blockOptsAddress := gorocksdb.NewDefaultBlockBasedTableOptions() - blockOptsAddress.SetBlockSize(16 << 10) // 16kB - blockOptsAddress.SetBlockCache(c) // 8GB - blockOptsAddress.SetCacheIndexAndFilterBlocks(true) - blockOptsAddress.SetPinL0FilterAndIndexBlocksInCache(true) - - optsAddresses := gorocksdb.NewDefaultOptions() - optsAddresses.SetBlockBasedTableFactory(blockOptsAddress) - optsAddresses.SetCreateIfMissing(true) - optsAddresses.SetCreateIfMissingColumnFamilies(true) - optsAddresses.SetMaxBackgroundCompactions(6) - optsAddresses.SetMaxBackgroundFlushes(6) - optsAddresses.SetBytesPerSync(1 << 20) // 1MB - optsAddresses.SetWriteBufferSize(1 << 27) // 128MB - optsAddresses.SetMaxOpenFiles(25000) - optsAddresses.SetCompression(gorocksdb.LZ4HCCompression) - + // opts with bloom filter + opts := createAndSetDBOptions(10, true, c) + // opts for addresses without bloom filter + // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter + optsAddresses := createAndSetDBOptions(0, true, c) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) if err != nil { return nil, nil, err @@ -111,7 +77,7 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column // needs to be called to release it. func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { glog.Infof("rocksdb: open %s, version %v", path, dbVersion) - c := gorocksdb.NewLRUCache(8 << 30) // 8GB + c := gorocksdb.NewLRUCache(1 << 30) // 1GB db, cfh, err := openDB(path, c) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() From 200131238263d463d41d26ff860ef3ddc0cdd9a3 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 23 Aug 2018 11:18:36 +0200 Subject: [PATCH 028/123] Extract bulk connect to own source file --- db/bulkconnect.go | 260 ++++++++++++++++++++++++++++++++++++++++++++++ db/rocksdb.go | 251 -------------------------------------------- 2 files changed, 260 insertions(+), 251 deletions(-) create mode 100644 db/bulkconnect.go diff --git a/db/bulkconnect.go b/db/bulkconnect.go new file mode 100644 index 00000000..ce1af0aa --- /dev/null +++ b/db/bulkconnect.go @@ -0,0 +1,260 @@ +package db + +import ( + "blockbook/bchain" + "time" + + "github.com/golang/glog" + "github.com/tecbot/gorocksdb" +) + +// bulk connect +// in bulk mode the data are cached and stored to db in batches +// it speeds up the import in two ways: +// 1) balances and txAddresses are modified several times during the import, there is a chance that the modifications are done before write to DB +// 2) rocksdb seems to handle better fewer larger batches than continuous stream of smaller batches + +type bulkAddresses struct { + height uint32 + addresses map[string][]outpoint +} + +// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type BulkConnect struct { + d *RocksDB + isUTXO bool + bulkAddresses []bulkAddresses + bulkAddressesCount int + txAddressesMap map[string]*TxAddresses + balances map[string]*AddrBalance + height uint32 +} + +const ( + maxBulkAddresses = 400000 + maxBulkTxAddresses = 1500000 + partialStoreAddresses = maxBulkTxAddresses / 10 + maxBulkBalances = 2500000 + partialStoreBalances = maxBulkBalances / 10 +) + +// InitBulkConnect initializes bulk connect and switches DB to inconsistent state +func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { + bc := &BulkConnect{ + d: d, + isUTXO: d.chainParser.IsUTXOChain(), + txAddressesMap: make(map[string]*TxAddresses), + balances: make(map[string]*AddrBalance), + } + if err := d.SetInconsistentState(true); err != nil { + return nil, err + } + glog.Info("rocksdb: bulk connect init, db set to inconsistent state") + return bc, nil +} + +func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int, int, error) { + var txm map[string]*TxAddresses + var sp int + if all { + txm = b.txAddressesMap + b.txAddressesMap = make(map[string]*TxAddresses) + } else { + txm = make(map[string]*TxAddresses) + for k, a := range b.txAddressesMap { + // store all completely spent transactions, they will not be modified again + r := true + for _, o := range a.Outputs { + if o.Spent == false { + r = false + break + } + } + if r { + txm[k] = a + delete(b.txAddressesMap, k) + } + } + sp = len(txm) + // store some other random transactions if necessary + if len(txm) < partialStoreAddresses { + for k, a := range b.txAddressesMap { + txm[k] = a + delete(b.txAddressesMap, k) + if len(txm) >= partialStoreAddresses { + break + } + } + } + } + if err := b.d.storeTxAddresses(wb, txm); err != nil { + return 0, 0, err + } + return len(txm), sp, nil +} + +func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) { + defer close(c) + start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + count, sp, err := b.storeTxAddresses(wb, all) + if err != nil { + c <- err + return + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + return + } + glog.Info("rocksdb: height ", b.height, ", stored ", count, " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + c <- nil +} + +func (b *BulkConnect) storeBalances(wb *gorocksdb.WriteBatch, all bool) (int, error) { + var bal map[string]*AddrBalance + if all { + bal = b.balances + b.balances = make(map[string]*AddrBalance) + } else { + bal = make(map[string]*AddrBalance) + // store some random balances + for k, a := range b.balances { + bal[k] = a + delete(b.balances, k) + if len(bal) >= partialStoreBalances { + break + } + } + } + if err := b.d.storeBalances(wb, bal); err != nil { + return 0, err + } + return len(bal), nil +} + +func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) { + defer close(c) + start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + count, err := b.storeBalances(wb, all) + if err != nil { + c <- err + return + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + return + } + glog.Info("rocksdb: height ", b.height, ", stored ", count, " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) + c <- nil +} + +func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { + for _, ba := range b.bulkAddresses { + if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { + return err + } + } + b.bulkAddressesCount = 0 + b.bulkAddresses = b.bulkAddresses[:0] + return nil +} + +// ConnectBlock connects block in bulk mode +func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { + b.height = block.Height + if !b.isUTXO { + return b.d.ConnectBlock(block) + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + addresses := make(map[string][]outpoint) + if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { + return err + } + start := time.Now() + var sa bool + var storeAddressesChan, storeBalancesChan chan error + if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + sa = true + if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { + storeAddressesChan = make(chan error) + go b.parallelStoreTxAddresses(storeAddressesChan, false) + } + if len(b.balances)+partialStoreBalances > maxBulkBalances { + storeBalancesChan = make(chan error) + go b.parallelStoreBalances(storeBalancesChan, false) + } + } + b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ + height: block.Height, + addresses: addresses, + }) + b.bulkAddressesCount += len(addresses) + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + } + if storeBlockTxs { + if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { + return err + } + } + if err := b.d.writeHeight(wb, block, opInsert); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + } + if storeAddressesChan != nil { + if err := <-storeAddressesChan; err != nil { + return err + } + } + if storeBalancesChan != nil { + if err := <-storeBalancesChan; err != nil { + return err + } + } + return nil +} + +// Close flushes the cached data and switches DB from inconsistent state open +// after Close, the BulkConnect cannot be used +func (b *BulkConnect) Close() error { + glog.Info("rocksdb: bulk connect closing") + start := time.Now() + storeAddressesChan := make(chan error) + go b.parallelStoreTxAddresses(storeAddressesChan, true) + storeBalancesChan := make(chan error) + go b.parallelStoreBalances(storeBalancesChan, true) + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + if err := <-storeAddressesChan; err != nil { + return err + } + if err := <-storeBalancesChan; err != nil { + return err + } + if err := b.d.SetInconsistentState(false); err != nil { + return err + } + glog.Info("rocksdb: bulk connect closed, db set to open state") + b.d = nil + return nil +} diff --git a/db/rocksdb.go b/db/rocksdb.go index 704f3a90..2a151ba4 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -273,257 +273,6 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return d.db.Write(d.wo, wb) } -// bulk connect -// in bulk mode the data are cached and stored to db in batches -// it speeds up the import in two ways: -// 1) balances and txAddresses are modified several times during the import, there is a chance that the modifications are done before write to DB -// 2) rocksdb seems to handle better fewer larger batches than continuous stream of smaller batches - -type bulkAddresses struct { - height uint32 - addresses map[string][]outpoint -} - -// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way -type BulkConnect struct { - d *RocksDB - isUTXO bool - bulkAddresses []bulkAddresses - bulkAddressesCount int - txAddressesMap map[string]*TxAddresses - balances map[string]*AddrBalance - height uint32 -} - -const ( - maxBulkAddresses = 400000 - maxBulkTxAddresses = 1500000 - partialStoreAddresses = maxBulkTxAddresses / 10 - maxBulkBalances = 2500000 - partialStoreBalances = maxBulkBalances / 10 -) - -// InitBulkConnect initializes bulk connect and switches DB to inconsistent state -func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { - bc := &BulkConnect{ - d: d, - isUTXO: d.chainParser.IsUTXOChain(), - txAddressesMap: make(map[string]*TxAddresses), - balances: make(map[string]*AddrBalance), - } - if err := d.SetInconsistentState(true); err != nil { - return nil, err - } - glog.Info("rocksdb: bulk connect init, db set to inconsistent state") - return bc, nil -} - -func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int, int, error) { - var txm map[string]*TxAddresses - var sp int - if all { - txm = b.txAddressesMap - b.txAddressesMap = make(map[string]*TxAddresses) - } else { - txm = make(map[string]*TxAddresses) - for k, a := range b.txAddressesMap { - // store all completely spent transactions, they will not be modified again - r := true - for _, o := range a.Outputs { - if o.Spent == false { - r = false - break - } - } - if r { - txm[k] = a - delete(b.txAddressesMap, k) - } - } - sp = len(txm) - // store some other random transactions if necessary - if len(txm) < partialStoreAddresses { - for k, a := range b.txAddressesMap { - txm[k] = a - delete(b.txAddressesMap, k) - if len(txm) >= partialStoreAddresses { - break - } - } - } - } - if err := b.d.storeTxAddresses(wb, txm); err != nil { - return 0, 0, err - } - return len(txm), sp, nil -} - -func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) { - defer close(c) - start := time.Now() - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - count, sp, err := b.storeTxAddresses(wb, all) - if err != nil { - c <- err - return - } - if err := b.d.db.Write(b.d.wo, wb); err != nil { - c <- err - return - } - glog.Info("rocksdb: height ", b.height, ", stored ", count, " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) - c <- nil -} - -func (b *BulkConnect) storeBalances(wb *gorocksdb.WriteBatch, all bool) (int, error) { - var bal map[string]*AddrBalance - if all { - bal = b.balances - b.balances = make(map[string]*AddrBalance) - } else { - bal = make(map[string]*AddrBalance) - // store some random balances - for k, a := range b.balances { - bal[k] = a - delete(b.balances, k) - if len(bal) >= partialStoreBalances { - break - } - } - } - if err := b.d.storeBalances(wb, bal); err != nil { - return 0, err - } - return len(bal), nil -} - -func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) { - defer close(c) - start := time.Now() - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - count, err := b.storeBalances(wb, all) - if err != nil { - c <- err - return - } - if err := b.d.db.Write(b.d.wo, wb); err != nil { - c <- err - return - } - glog.Info("rocksdb: height ", b.height, ", stored ", count, " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) - c <- nil -} - -func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { - for _, ba := range b.bulkAddresses { - if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { - return err - } - } - b.bulkAddressesCount = 0 - b.bulkAddresses = b.bulkAddresses[:0] - return nil -} - -// ConnectBlock connects block in bulk mode -func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { - b.height = block.Height - if !b.isUTXO { - return b.d.ConnectBlock(block) - } - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - addresses := make(map[string][]outpoint) - if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { - return err - } - start := time.Now() - var sa bool - var storeAddressesChan, storeBalancesChan chan error - if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { - sa = true - if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { - storeAddressesChan = make(chan error) - go b.parallelStoreTxAddresses(storeAddressesChan, false) - } - if len(b.balances)+partialStoreBalances > maxBulkBalances { - storeBalancesChan = make(chan error) - go b.parallelStoreBalances(storeBalancesChan, false) - } - } - b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ - height: block.Height, - addresses: addresses, - }) - b.bulkAddressesCount += len(addresses) - bac := b.bulkAddressesCount - if sa || b.bulkAddressesCount > maxBulkAddresses { - if err := b.storeBulkAddresses(wb); err != nil { - return err - } - } - if storeBlockTxs { - if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { - return err - } - } - if err := b.d.writeHeight(wb, block, opInsert); err != nil { - return err - } - if err := b.d.db.Write(b.d.wo, wb); err != nil { - return err - } - if bac > b.bulkAddressesCount { - glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) - } - if storeAddressesChan != nil { - if err := <-storeAddressesChan; err != nil { - return err - } - } - if storeBalancesChan != nil { - if err := <-storeBalancesChan; err != nil { - return err - } - } - return nil -} - -// Close flushes the cached data and switches DB from inconsistent state open -// after Close, the BulkConnect cannot be used -func (b *BulkConnect) Close() error { - glog.Info("rocksdb: bulk connect closing") - start := time.Now() - storeAddressesChan := make(chan error) - go b.parallelStoreTxAddresses(storeAddressesChan, true) - storeBalancesChan := make(chan error) - go b.parallelStoreBalances(storeBalancesChan, true) - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - bac := b.bulkAddressesCount - if err := b.storeBulkAddresses(wb); err != nil { - return err - } - if err := b.d.db.Write(b.d.wo, wb); err != nil { - return err - } - glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) - if err := <-storeAddressesChan; err != nil { - return err - } - if err := <-storeBalancesChan; err != nil { - return err - } - if err := b.d.SetInconsistentState(false); err != nil { - return err - } - glog.Info("rocksdb: bulk connect closed, db set to open state") - b.d = nil - return nil -} - // Addresses index type outpoint struct { From 05f4fb779540a5b6bfad5fd100287dab9c63723d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 23 Aug 2018 13:42:43 +0200 Subject: [PATCH 029/123] Pin specific rocksdb version (rocksdb-5.14.3) in build --- build/docker/bin/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/docker/bin/Dockerfile b/build/docker/bin/Dockerfile index 89047bb3..b957ad16 100644 --- a/build/docker/bin/Dockerfile +++ b/build/docker/bin/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update && \ apt-get clean ENV GOLANG_VERSION=go1.10.linux-amd64 +ENV ROCKSDB_VERSION=rocksdb-5.14.3 ENV GOPATH=/go ENV PATH=$PATH:$GOPATH/bin ENV CGO_CFLAGS="-I/opt/rocksdb/include" @@ -26,7 +27,7 @@ RUN echo -n "GO version: " && go version RUN echo -n "GOPATH: " && echo $GOPATH # install rocksdb -RUN cd /opt && git clone https://github.com/facebook/rocksdb.git +RUN cd /opt && git clone -b $ROCKSDB_VERSION --depth 1 https://github.com/facebook/rocksdb.git RUN cd /opt/rocksdb && CFLAGS=-fPIC CXXFLAGS=-fPIC make -j 4 release RUN strip /opt/rocksdb/ldb /opt/rocksdb/sst_dump && \ cp /opt/rocksdb/ldb /opt/rocksdb/sst_dump /build From 328312e48adcc7fda0fbda1cd86e47d1a194bf60 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 23 Aug 2018 21:14:16 +0200 Subject: [PATCH 030/123] Add dbcache configuration flag, tune db options --- blockbook.go | 5 +-- configs/coins/bitcoin.json | 4 +-- configs/coins/bitcoin_testnet.json | 2 +- db/bulkconnect.go | 6 ++-- db/dboptions.go | 53 ++++++++++++++---------------- db/rocksdb.go | 8 ++--- db/rocksdb_test.go | 2 +- 7 files changed, 38 insertions(+), 42 deletions(-) diff --git a/blockbook.go b/blockbook.go index 7e3da52b..07e507d5 100644 --- a/blockbook.go +++ b/blockbook.go @@ -38,7 +38,8 @@ const storeInternalStatePeriodMs = 59699 var ( blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") - dbPath = flag.String("datadir", "./data", "path to database directory") + dbPath = flag.String("datadir", "./data", "path to database directory") + dbCache = flag.Int("dbcache", 1<<30, "size of the rocksdb cache") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") @@ -162,7 +163,7 @@ func main() { glog.Fatal("rpc: ", err) } - index, err = db.NewRocksDB(*dbPath, chain.GetChainParser(), metrics) + index, err = db.NewRocksDB(*dbPath, *dbCache, chain.GetChainParser(), metrics) if err != nil { glog.Fatal("rocksDB: ", err) } diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index d5435bba..a957102a 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -47,8 +47,8 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btc-explorer.trezor.io/", - "additional_params": "", + "explorer_url": "/explorer", + "additional_params": "-dbcache=4294967296", "block_chain": { "parse": true, "mempool_workers": 8, diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index 3397cb95..760163be 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -47,7 +47,7 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btc-testnet-explorer.trezor.io/", + "explorer_url": "/explorer", "additional_params": "", "block_chain": { "parse": true, diff --git a/db/bulkconnect.go b/db/bulkconnect.go index ce1af0aa..bece9c6b 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -31,10 +31,10 @@ type BulkConnect struct { } const ( - maxBulkAddresses = 400000 - maxBulkTxAddresses = 1500000 + maxBulkAddresses = 200000 + maxBulkTxAddresses = 500000 partialStoreAddresses = maxBulkTxAddresses / 10 - maxBulkBalances = 2500000 + maxBulkBalances = 800000 partialStoreBalances = maxBulkBalances / 10 ) diff --git a/db/dboptions.go b/db/dboptions.go index 81ff82d3..2e0da942 100644 --- a/db/dboptions.go +++ b/db/dboptions.go @@ -1,47 +1,51 @@ package db -// #include "rocksdb/c.h" -import "C" import ( - "reflect" - "unsafe" - "github.com/tecbot/gorocksdb" ) -func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache) *gorocksdb.Options { - // get access to all db options, gorocksdb does not expose all +/* + possible additional tuning, using options not accessible by gorocksdb + +// #include "rocksdb/c.h" +import "C" - // opts := gorocksdb.NewDefaultOptions() cNativeOpts := C.rocksdb_options_create() opts := &gorocksdb.Options{} cField := reflect.Indirect(reflect.ValueOf(opts)).FieldByName("c") cPtr := (**C.rocksdb_options_t)(unsafe.Pointer(cField.UnsafeAddr())) *cPtr = cNativeOpts - // blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() cNativeBlockOpts := C.rocksdb_block_based_options_create() blockOpts := &gorocksdb.BlockBasedTableOptions{} cBlockField := reflect.Indirect(reflect.ValueOf(blockOpts)).FieldByName("c") cBlockPtr := (**C.rocksdb_block_based_table_options_t)(unsafe.Pointer(cBlockField.UnsafeAddr())) *cBlockPtr = cNativeBlockOpts + // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters + blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) + C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true)) + C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096)) + C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true)) + blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) + +// boolToChar converts a bool value to C.uchar. +func boolToChar(b bool) C.uchar { + if b { + return 1 + } + return 0 +} +*/ + +func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options { + blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() blockOpts.SetBlockSize(32 << 10) // 32kB blockOpts.SetBlockCache(c) if bloomBits > 0 { blockOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(bloomBits)) } - - // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters - if twoLevelIndex { - // blockOpts.SetCacheIndexAndFilterBlocks(true) - blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) - C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true)) - C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096)) - C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true)) - blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) - } - + opts := gorocksdb.NewDefaultOptions() opts.SetBlockBasedTableFactory(blockOpts) opts.SetCreateIfMissing(true) opts.SetCreateIfMissingColumnFamilies(true) @@ -52,14 +56,5 @@ func createAndSetDBOptions(bloomBits int, twoLevelIndex bool, c *gorocksdb.Cache opts.SetMaxBytesForLevelBase(1 << 27) // 128MB opts.SetMaxOpenFiles(25000) opts.SetCompression(gorocksdb.LZ4HCCompression) - return opts } - -// boolToChar converts a bool value to C.uchar. -func boolToChar(b bool) C.uchar { - if b { - return 1 - } - return 0 -} diff --git a/db/rocksdb.go b/db/rocksdb.go index 2a151ba4..bceb1b4d 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -60,10 +60,10 @@ var cfNames = []string{"default", "height", "addresses", "txAddresses", "address func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter - opts := createAndSetDBOptions(10, true, c) + opts := createAndSetDBOptions(10, c) // opts for addresses without bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter - optsAddresses := createAndSetDBOptions(0, true, c) + optsAddresses := createAndSetDBOptions(0, c) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) @@ -75,9 +75,9 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { +func NewRocksDB(path string, cacheSize int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { glog.Infof("rocksdb: open %s, version %v", path, dbVersion) - c := gorocksdb.NewLRUCache(1 << 30) // 1GB + c := gorocksdb.NewLRUCache(cacheSize) db, cfh, err := openDB(path, c) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index e74d0664..e36a3ab0 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -37,7 +37,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - d, err := NewRocksDB(tmp, p, nil) + d, err := NewRocksDB(tmp, 100000, p, nil) if err != nil { t.Fatal(err) } From ad5ddbd02961663525e1c9f222bc8664cae3584e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 23 Aug 2018 23:20:07 +0200 Subject: [PATCH 031/123] Write height column in bulk mode --- api/worker.go | 10 ++++----- db/bulkconnect.go | 52 +++++++++++++++++++++++++++------------------- db/rocksdb.go | 41 ++++++++++++++++++++++-------------- db/rocksdb_test.go | 9 ++++---- 4 files changed, 65 insertions(+), 47 deletions(-) diff --git a/api/worker.go b/api/worker.go index b0ddfe89..f8867c63 100644 --- a/api/worker.go +++ b/api/worker.go @@ -181,7 +181,7 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) valInSat.Add(&valInSat, &vin.ValueSat) a, err := tai.Addresses(w.chainParser) - if err != nil && len(a) == 1 { + if err == nil && len(a) == 1 { vin.Addr = a[0] } } @@ -194,7 +194,7 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn vout.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) valOutSat.Add(&valOutSat, &vout.ValueSat) a, err := tao.Addresses(w.chainParser) - if err != nil { + if err == nil { vout.ScriptPubKey.Addresses = a } } @@ -204,12 +204,12 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn feesSat.SetUint64(0) } r := &Tx{ - Blockhash: bi.BlockHash, + Blockhash: bi.Hash, Blockheight: int(ta.Height), - Blocktime: bi.Time.Unix(), + Blocktime: bi.Time, Confirmations: bestheight - ta.Height + 1, Fees: w.chainParser.AmountToDecimalString(&feesSat), - Time: bi.Time.Unix(), + Time: bi.Time, Txid: txid, ValueIn: w.chainParser.AmountToDecimalString(&valInSat), ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), diff --git a/db/bulkconnect.go b/db/bulkconnect.go index bece9c6b..eeefd234 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -16,6 +16,7 @@ import ( type bulkAddresses struct { height uint32 + bi BlockInfo addresses map[string][]outpoint } @@ -156,6 +157,9 @@ func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { return err } + if err := b.d.writeHeight(wb, ba.height, &ba.bi, opInsert); err != nil { + return err + } } b.bulkAddressesCount = 0 b.bulkAddresses = b.bulkAddresses[:0] @@ -168,15 +172,12 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro if !b.isUTXO { return b.d.ConnectBlock(block) } - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() addresses := make(map[string][]outpoint) if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { return err } - start := time.Now() - var sa bool var storeAddressesChan, storeBalancesChan chan error + var sa bool if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { sa = true if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { @@ -189,30 +190,39 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro } } b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ - height: block.Height, + height: block.Height, + bi: BlockInfo{ + Hash: block.Hash, + Time: block.Time, + Txs: uint32(len(block.Txs)), + Size: uint32(block.Size), + }, addresses: addresses, }) b.bulkAddressesCount += len(addresses) - bac := b.bulkAddressesCount - if sa || b.bulkAddressesCount > maxBulkAddresses { - if err := b.storeBulkAddresses(wb); err != nil { + // open WriteBatch only if going to write + if sa || b.bulkAddressesCount > maxBulkAddresses || storeBlockTxs { + start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + } + if storeBlockTxs { + if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { + return err + } + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { return err } - } - if storeBlockTxs { - if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { - return err + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) } } - if err := b.d.writeHeight(wb, block, opInsert); err != nil { - return err - } - if err := b.d.db.Write(b.d.wo, wb); err != nil { - return err - } - if bac > b.bulkAddressesCount { - glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) - } if storeAddressesChan != nil { if err := <-storeAddressesChan; err != nil { return err diff --git a/db/rocksdb.go b/db/rocksdb.go index bceb1b4d..de498388 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -237,7 +237,7 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { isUTXO := d.chainParser.IsUTXOChain() - if err := d.writeHeight(wb, block, op); err != nil { + if err := d.writeHeightFromBlock(wb, block, op); err != nil { return err } if isUTXO { @@ -862,14 +862,15 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // Block index +// BlockInfo holds information about blocks kept in column height type BlockInfo struct { - BlockHash string - Time time.Time - Txs uint32 - Size uint32 + Hash string + Time int64 + Txs uint32 + Size uint32 } -func (d *RocksDB) packBlockInfo(block *bchain.Block) ([]byte, error) { +func (d *RocksDB) packBlockInfo(block *BlockInfo) ([]byte, error) { packed := make([]byte, 0, 64) varBuf := make([]byte, vlq.MaxLen64) b, err := d.chainParser.PackBlockHash(block.Hash) @@ -878,7 +879,7 @@ func (d *RocksDB) packBlockInfo(block *bchain.Block) ([]byte, error) { } packed = append(packed, b...) packed = append(packed, packUint(uint32(block.Time))...) - l := packVaruint(uint(len(block.Txs)), varBuf) + l := packVaruint(uint(block.Txs), varBuf) packed = append(packed, varBuf[:l]...) l = packVaruint(uint(block.Size), varBuf) packed = append(packed, varBuf[:l]...) @@ -899,10 +900,10 @@ func (d *RocksDB) unpackBlockInfo(buf []byte) (*BlockInfo, error) { txs, l := unpackVaruint(buf[pl+4:]) size, _ := unpackVaruint(buf[pl+4+l:]) return &BlockInfo{ - BlockHash: txid, - Time: time.Unix(int64(t), 0), - Txs: uint32(txs), - Size: uint32(size), + Hash: txid, + Time: int64(t), + Txs: uint32(txs), + Size: uint32(size), }, nil } @@ -917,7 +918,7 @@ func (d *RocksDB) GetBestBlock() (uint32, string, error) { if glog.V(1) { glog.Infof("rocksdb: bestblock %d %+v", bestHeight, info) } - return bestHeight, info.BlockHash, err + return bestHeight, info.Hash, err } } return 0, "", nil @@ -935,7 +936,7 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { if info == nil { return "", err } - return info.BlockHash, nil + return info.Hash, nil } // GetBlockInfo returns block info stored in db @@ -949,12 +950,20 @@ func (d *RocksDB) GetBlockInfo(height uint32) (*BlockInfo, error) { return d.unpackBlockInfo(val.Data()) } -func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - key := packUint(block.Height) +func (d *RocksDB) writeHeightFromBlock(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { + return d.writeHeight(wb, block.Height, &BlockInfo{ + Hash: block.Hash, + Time: block.Time, + Txs: uint32(len(block.Txs)), + Size: uint32(block.Size), + }, op) +} +func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, height uint32, bi *BlockInfo, op int) error { + key := packUint(height) switch op { case opInsert: - val, err := d.packBlockInfo(block) + val, err := d.packBlockInfo(bi) if err != nil { return err } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index e36a3ab0..3b21b93d 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -16,7 +16,6 @@ import ( "sort" "strings" "testing" - "time" vlq "github.com/bsm/go-vlq" "github.com/juju/errors" @@ -731,10 +730,10 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } iw := &BlockInfo{ - BlockHash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", - Txs: 4, - Size: 2345678, - Time: time.Unix(1534859123, 0), + Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Txs: 4, + Size: 2345678, + Time: 1534859123, } if !reflect.DeepEqual(info, iw) { t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) From 7d708ef868bdd1fb7d8ba906555b230cd726f769 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 24 Aug 2018 16:17:43 +0200 Subject: [PATCH 032/123] Unify handling and error handling of pages in public interface --- api/types.go | 16 +++ api/worker.go | 33 ++--- blockbook.go | 6 +- server/public.go | 278 +++++++++++++++++++++++------------- static/templates/error.html | 4 + 5 files changed, 220 insertions(+), 117 deletions(-) create mode 100644 static/templates/error.html diff --git a/api/types.go b/api/types.go index be289bd6..d781b5d8 100644 --- a/api/types.go +++ b/api/types.go @@ -2,6 +2,22 @@ package api import "math/big" +type ApiError struct { + Text string + Public bool +} + +func (e *ApiError) Error() string { + return e.Text +} + +func NewApiError(s string, public bool) error { + return &ApiError{ + Text: s, + Public: public, + } +} + type ScriptSig struct { Hex string `json:"hex"` Asm string `json:"asm,omitempty"` diff --git a/api/worker.go b/api/worker.go index f8867c63..afcfb19c 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,10 +4,11 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" - "errors" "math/big" + "time" "github.com/golang/glog" + "github.com/juju/errors" ) // Worker is handle to api worker @@ -35,13 +36,13 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) (*Tx, error) { bchainTx, height, err := w.txCache.GetTransaction(txid, bestheight) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "txCache.GetTransaction %v", txid) } var blockhash string if bchainTx.Confirmations > 0 { blockhash, err = w.db.GetBlockHash(height) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetBlockHash %v", height) } } var valInSat, valOutSat, feesSat big.Int @@ -57,7 +58,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) if bchainVin.Txid != "" { otx, _, err := w.txCache.GetTransaction(bchainVin.Txid, bestheight) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "txCache.GetTransaction %v", bchainVin.Txid) } if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] @@ -220,28 +221,28 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn } // GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddress(address string, page int, txsOnPage int) (*Address, error) { - glog.Info(address, " start") +func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids bool) (*Address, error) { + start := time.Now() ba, err := w.db.GetAddressBalance(address) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetAddressBalance %v", address) } if ba == nil { - return nil, errors.New("Address not found") + return nil, NewApiError("Address not found", true) } txc, err := w.getAddressTxids(address, false) - txc = UniqueTxidsInReverse(txc) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "getAddressTxids %v false", address) } + txc = UniqueTxidsInReverse(txc) txm, err := w.getAddressTxids(address, true) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "getAddressTxids %v true", address) } txm = UniqueTxidsInReverse(txm) bestheight, _, err := w.db.GetBestBlock() if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetBestBlock") } // paging if page < 0 { @@ -268,7 +269,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int) (*Address, tx, err := w.GetTransaction(tx, bestheight, false) // mempool transaction may fail if err != nil { - glog.Error("GetTransaction ", tx, ": ", err) + glog.Error("GetTransaction in mempool ", tx, ": ", err) } else { uBalSat.Sub(tx.getAddrVoutValue(address), tx.getAddrVinValue(address)) txs[txi] = tx @@ -282,7 +283,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int) (*Address, txid := txc[i] ta, err := w.db.GetTxAddresses(txid) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) } if ta == nil { glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") @@ -290,7 +291,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int) (*Address, } bi, err := w.db.GetBlockInfo(ta.Height) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) } if bi == nil { glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") @@ -312,7 +313,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int) (*Address, TotalPages: totalPages, TxsOnPage: txsOnPage, } - glog.Info(address, " finished") + glog.Info(address, " finished in ", time.Since(start)) return r, nil } diff --git a/blockbook.go b/blockbook.go index 07e507d5..547538c9 100644 --- a/blockbook.go +++ b/blockbook.go @@ -55,6 +55,8 @@ var ( syncWorkers = flag.Int("workers", 8, "number of workers to process blocks") dryRun = flag.Bool("dryrun", false, "do not index blocks, only download") + debugMode = flag.Bool("debug", false, "debug mode, return more verbose errors, reload templates on each request") + internalBinding = flag.String("internal", "", "internal http server binding [address]:port, (default no internal server)") publicBinding = flag.String("public", "", "public http server binding [address]:port[/path], (default no public server)") @@ -130,7 +132,7 @@ func main() { chanOsSignal = make(chan os.Signal, 1) signal.Notify(chanOsSignal, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) - glog.Infof("Blockbook: %+v", common.GetVersionInfo()) + glog.Infof("Blockbook: %+v, debug mode %v", common.GetVersionInfo(), *debugMode) if *prof != "" { go func() { @@ -270,7 +272,7 @@ func main() { var publicServer *server.PublicServer if *publicBinding != "" { - publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, txCache, *explorerURL, metrics, internalState) + publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, txCache, *explorerURL, metrics, internalState, *debugMode) if err != nil { glog.Error("socketio: ", err) return diff --git a/server/public.go b/server/public.go index abe67790..36556850 100644 --- a/server/public.go +++ b/server/public.go @@ -10,6 +10,8 @@ import ( "fmt" "html/template" "net/http" + "reflect" + "runtime" "strconv" "strings" "time" @@ -35,12 +37,12 @@ type PublicServer struct { explorerURL string metrics *common.Metrics is *common.InternalState - txTpl *template.Template - addressTpl *template.Template + templates []*template.Template + debug bool } // NewPublicServer creates new public server http interface to blockbook and returns its handle -func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState) (*PublicServer, error) { +func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState, debugMode bool) (*PublicServer, error) { api, err := api.NewWorker(db, chain, txCache, is) if err != nil { @@ -72,55 +74,34 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch explorerURL: explorerURL, metrics: metrics, is: is, + debug: debugMode, } // favicon serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) // support for tests of socket.io interface serveMux.Handle(path+"test.html", http.FileServer(http.Dir("./static/"))) - // redirect to Bitcore for details of transaction + // redirect to wallet requests for tx and address, possibly to external site serveMux.HandleFunc(path+"tx/", s.txRedirect) serveMux.HandleFunc(path+"address/", s.addressRedirect) // explorer - serveMux.HandleFunc(path+"explorer/tx/", s.explorerTx) - serveMux.HandleFunc(path+"explorer/address/", s.explorerAddress) + serveMux.HandleFunc(path+"explorer/tx/", s.htmlTemplateHandler(s.explorerTx)) + serveMux.HandleFunc(path+"explorer/address/", s.htmlTemplateHandler(s.explorerAddress)) serveMux.Handle(path+"static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))) // API calls - serveMux.HandleFunc(path+"api/block-index/", s.apiBlockIndex) - serveMux.HandleFunc(path+"api/tx/", s.apiTx) - serveMux.HandleFunc(path+"api/address/", s.apiAddress) + serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex)) + serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx)) + serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress)) // handle socket.io serveMux.Handle(path+"socket.io/", socketio.GetHandler()) // default handler serveMux.HandleFunc(path, s.index) - s.txTpl, s.addressTpl = parseTemplates() + s.templates = parseTemplates() return s, nil } -func parseTemplates() (txTpl, addressTpl *template.Template) { - templateFuncMap := template.FuncMap{ - "formatUnixTime": formatUnixTime, - "formatAmount": formatAmount, - "setTxToTemplateData": setTxToTemplateData, - "stringInSlice": stringInSlice, - } - txTpl = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) - addressTpl = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/base.html")) - return -} - -func formatUnixTime(ut int64) string { - return time.Unix(ut, 0).Format(time.RFC1123) -} - -// for now return the string as it is -// in future could be used to do coin specific formatting -func formatAmount(a string) string { - return a -} - // Run starts the server func (s *PublicServer) Run() error { if s.certFiles == "" { @@ -153,6 +134,20 @@ func (s *PublicServer) OnNewTxAddr(txid string, addr string) { s.socketio.OnNewTxAddr(txid, addr) } +func (s *PublicServer) txRedirect(w http.ResponseWriter, r *http.Request) { + if s.explorerURL != "" { + http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) + s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() + } +} + +func (s *PublicServer) addressRedirect(w http.ResponseWriter, r *http.Request) { + if s.explorerURL != "" { + http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) + s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() + } +} + func splitBinding(binding string) (addr string, path string) { i := strings.Index(binding, "/") if i >= 0 { @@ -171,34 +166,154 @@ func joinURL(base string, part string) string { return part } -func (s *PublicServer) txRedirect(w http.ResponseWriter, r *http.Request) { - if s.explorerURL != "" { - http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) - s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() +func getFunctionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} + +func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, error)) func(w http.ResponseWriter, r *http.Request) { + type jsonError struct { + Error string `json:"error"` + } + return func(w http.ResponseWriter, r *http.Request) { + var data interface{} + var err error + defer func() { + if e := recover(); e != nil { + glog.Error(getFunctionName(handler), " recovered from panic: ", e) + if s.debug { + data = jsonError{fmt.Sprint("Internal server error: recovered from panic ", e)} + } else { + data = jsonError{"Internal server error"} + } + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + json.NewEncoder(w).Encode(data) + }() + data, err = handler(r) + if err != nil || data == nil { + if apiErr, ok := err.(*api.ApiError); ok { + data = jsonError{apiErr.Error()} + } else { + if err != nil { + glog.Error(getFunctionName(handler), " error: ", err) + } + if s.debug { + data = jsonError{fmt.Sprintf("Internal server error: %v, data %+v", err, data)} + } else { + data = jsonError{"Internal server error"} + } + } + } } } -func (s *PublicServer) addressRedirect(w http.ResponseWriter, r *http.Request) { - if s.explorerURL != "" { - http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) - s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() +func (s *PublicServer) newTemplateData() *TemplateData { + return &TemplateData{ + CoinName: s.is.Coin, + CoinShortcut: s.is.CoinShortcut, } } +func (s *PublicServer) newTemplateDataWithError(text string) *TemplateData { + return &TemplateData{ + CoinName: s.is.Coin, + CoinShortcut: s.is.CoinShortcut, + Error: &api.ApiError{Text: text}, + } +} +func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + var t tpl + var data *TemplateData + var err error + defer func() { + if e := recover(); e != nil { + glog.Error(getFunctionName(handler), " recovered from panic: ", e) + t = errorTpl + if s.debug { + data = s.newTemplateDataWithError(fmt.Sprint("Internal server error: recovered from panic ", e)) + } else { + data = s.newTemplateDataWithError("Internal server error") + } + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := s.templates[t].ExecuteTemplate(w, "base.html", data); err != nil { + glog.Error(err) + } + }() + if s.debug { + // reload templates on each request + // to reflect changes during development + s.templates = parseTemplates() + } + t, data, err = handler(r) + if err != nil || data == nil { + t = errorTpl + if apiErr, ok := err.(*api.ApiError); ok { + data = s.newTemplateData() + data.Error = apiErr + } else { + if err != nil { + glog.Error(getFunctionName(handler), " error: ", err) + } + if s.debug { + data = s.newTemplateDataWithError(fmt.Sprintf("Internal server error: %v, data %+v", err, data)) + } else { + data = s.newTemplateDataWithError("Internal server error") + } + } + } + } +} + +type tpl int + +const ( + errorTpl = tpl(iota) + txTpl + addressTpl +) + type TemplateData struct { CoinName string CoinShortcut string Address *api.Address AddrStr string Tx *api.Tx + Error *api.ApiError } +func parseTemplates() []*template.Template { + templateFuncMap := template.FuncMap{ + "formatUnixTime": formatUnixTime, + "formatAmount": formatAmount, + "setTxToTemplateData": setTxToTemplateData, + "stringInSlice": stringInSlice, + } + t := make([]*template.Template, 3) + t[errorTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) + t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) + t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/base.html")) + return t +} + +func formatUnixTime(ut int64) string { + return time.Unix(ut, 0).Format(time.RFC1123) +} + +// for now return the string as it is +// in future could be used to do coin specific formatting +func formatAmount(a string) string { + return a +} + +// called from template to support txdetail.html functionality func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { td.Tx = tx return td } -func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] @@ -207,26 +322,15 @@ func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) { tx, err = s.api.GetTransaction(txid, bestheight, true) } if err != nil { - glog.Error(err) + return errorTpl, nil, err } } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - - // temporarily reread the template on each request - // to reflect changes during development - s.txTpl, s.addressTpl = parseTemplates() - - data := &TemplateData{ - CoinName: s.is.Coin, - CoinShortcut: s.is.CoinShortcut, - Tx: tx, - } - if err := s.txTpl.ExecuteTemplate(w, "base.html", data); err != nil { - glog.Error(err) - } + data := s.newTemplateData() + data.Tx = tx + return txTpl, data, nil } -func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, error) { var address *api.Address var err error if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { @@ -235,27 +339,15 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) { page = 0 } addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page, txsOnPage) + address, err = s.api.GetAddress(addrID, page, txsOnPage, false) if err != nil { - glog.Error(err) - // TODO return error.html + return errorTpl, nil, err } } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - - // temporarily reread the template on each request - // to reflect changes during development - s.txTpl, s.addressTpl = parseTemplates() - - data := &TemplateData{ - CoinName: s.is.Coin, - CoinShortcut: s.is.CoinShortcut, - AddrStr: address.AddrStr, - Address: address, - } - if err := s.addressTpl.ExecuteTemplate(w, "base.html", data); err != nil { - glog.Error(err) - } + data := s.newTemplateData() + data.AddrStr = address.AddrStr + data.Address = address + return addressTpl, data, nil } type resAboutBlockbookPublic struct { @@ -272,6 +364,7 @@ type resAboutBlockbookPublic struct { About string `json:"about"` } +// TODO - this is temporary, return html status page func (s *PublicServer) index(w http.ResponseWriter, r *http.Request) { vi := common.GetVersionInfo() ss, bh, st := s.is.GetSyncState() @@ -297,7 +390,7 @@ func (s *PublicServer) index(w http.ResponseWriter, r *http.Request) { w.Write(buf) } -func (s *PublicServer) apiBlockIndex(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) apiBlockIndex(r *http.Request) (interface{}, error) { type resBlockIndex struct { BlockHash string `json:"blockHash"` About string `json:"about"` @@ -317,17 +410,15 @@ func (s *PublicServer) apiBlockIndex(w http.ResponseWriter, r *http.Request) { } if err != nil { glog.Error(err) - } else { - r := resBlockIndex{ - BlockHash: hash, - About: blockbookAbout, - } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(r) + return nil, err } + return resBlockIndex{ + BlockHash: hash, + About: blockbookAbout, + }, nil } -func (s *PublicServer) apiTx(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) { var tx *api.Tx var err error if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { @@ -335,17 +426,12 @@ func (s *PublicServer) apiTx(w http.ResponseWriter, r *http.Request) { bestheight, _, err := s.db.GetBestBlock() if err == nil { tx, err = s.api.GetTransaction(txid, bestheight, true) - } else { - glog.Error(err) } } - if err == nil { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(tx) - } + return tx, err } -func (s *PublicServer) apiAddress(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { var address *api.Address var err error if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { @@ -354,13 +440,7 @@ func (s *PublicServer) apiAddress(w http.ResponseWriter, r *http.Request) { page = 0 } addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page, txsInAPI) - if err != nil { - glog.Error(err) - } - } - if err == nil { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(address) + address, err = s.api.GetAddress(addrID, page, txsInAPI, true) } + return address, err } diff --git a/static/templates/error.html b/static/templates/error.html new file mode 100644 index 00000000..7ad0ad31 --- /dev/null +++ b/static/templates/error.html @@ -0,0 +1,4 @@ +{{define "specific"}} +

Error

+

{{.Error.Text}}

+{{end}} \ No newline at end of file From 16275601f41a734ed67b8fe49a02e5057e3f1fd0 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 26 Aug 2018 18:33:07 +0200 Subject: [PATCH 033/123] Decrease default db cache size to reduce memory footprint --- blockbook.go | 2 +- configs/coins/bitcoin.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/blockbook.go b/blockbook.go index 547538c9..b5b99552 100644 --- a/blockbook.go +++ b/blockbook.go @@ -39,7 +39,7 @@ var ( blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") dbPath = flag.String("datadir", "./data", "path to database directory") - dbCache = flag.Int("dbcache", 1<<30, "size of the rocksdb cache") + dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index a957102a..ec93e446 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -48,7 +48,7 @@ "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", "explorer_url": "/explorer", - "additional_params": "-dbcache=4294967296", + "additional_params": "-dbcache=1073741824", "block_chain": { "parse": true, "mempool_workers": 8, From b1e749dab9b9cd83a3e84de1525d2a9c4188424d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 26 Aug 2018 21:29:10 +0200 Subject: [PATCH 034/123] Add logging of memory status --- blockbook.go | 6 ++++++ db/sync.go | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/blockbook.go b/blockbook.go index b5b99552..f3f91b31 100644 --- a/blockbook.go +++ b/blockbook.go @@ -431,6 +431,8 @@ func storeInternalStateLoop() { lastCompute := time.Now() // randomize the duration between ComputeInternalStateColumnStats to avoid peaks after reboot of machine with multiple blockbooks computePeriod := 9*time.Hour + time.Duration(rand.Float64()*float64((2*time.Hour).Nanoseconds())) + lastLogMemory := time.Now() + logMemoryPeriod := 15 * time.Minute glog.Info("storeInternalStateLoop starting with db stats recompute period ", computePeriod) tickAndDebounce(storeInternalStatePeriodMs*time.Millisecond, (storeInternalStatePeriodMs-1)*time.Millisecond, chanStoreInternalState, func() { if !computeRunning && lastCompute.Add(computePeriod).Before(time.Now()) { @@ -447,6 +449,10 @@ func storeInternalStateLoop() { if err := index.StoreInternalState(internalState); err != nil { glog.Error("storeInternalStateLoop ", errors.ErrorStack(err)) } + if lastLogMemory.Add(logMemoryPeriod).Before(time.Now()) { + glog.Info(index.GetMemoryStats()) + lastLogMemory = time.Now() + } }) glog.Info("storeInternalStateLoop stopped") } diff --git a/db/sync.go b/db/sync.go index 8f52e589..6d5fd713 100644 --- a/db/sync.go +++ b/db/sync.go @@ -308,7 +308,8 @@ ConnectLoop: } if msTime.Before(time.Now()) { glog.Info(w.db.GetMemoryStats()) - msTime = time.Now().Add(1 * time.Minute) + w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) + msTime = time.Now().Add(10 * time.Minute) } h++ } From b56b6c7a9bb5712a4110d9d2804d16efe98bb515 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 27 Aug 2018 15:36:33 +0200 Subject: [PATCH 035/123] Get data for explorer using index v3 - WIP --- api/types.go | 26 +++++++------ api/worker.go | 77 +++++++++++++++++++++++++------------ static/css/main.css | 15 +++++++- static/templates/error.html | 2 +- 4 files changed, 82 insertions(+), 38 deletions(-) diff --git a/api/types.go b/api/types.go index d781b5d8..e5c734e4 100644 --- a/api/types.go +++ b/api/types.go @@ -30,6 +30,7 @@ type Vin struct { N int `json:"n"` ScriptSig ScriptSig `json:"scriptSig"` Addr string `json:"addr"` + AddrLink bool `json:"-"` Value string `json:"value"` ValueSat big.Int `json:"-"` } @@ -38,6 +39,7 @@ type ScriptPubKey struct { Hex string `json:"hex"` Asm string `json:"asm,omitempty"` Addresses []string `json:"addresses"` + AddrsLink []bool `json:"-"` Type string `json:"type,omitempty"` } type Vout struct { @@ -45,6 +47,7 @@ type Vout struct { ValueSat big.Int `json:"-"` N int `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` + Spent bool `json:"-"` SpentTxID string `json:"spentTxId,omitempty"` SpentIndex int `json:"spentIndex,omitempty"` SpentHeight int `json:"spentHeight,omitempty"` @@ -69,15 +72,16 @@ type Tx struct { } type Address struct { - AddrStr string `json:"addrStr"` - Balance string `json:"balance"` - TotalReceived string `json:"totalReceived"` - TotalSent string `json:"totalSent"` - UnconfirmedBalance string `json:"unconfirmedBalance"` - UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` - TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"transactions"` - Page int `json:"page"` - TotalPages int `json:"totalPages"` - TxsOnPage int `json:"txsOnPage"` + AddrStr string `json:"addrStr"` + Balance string `json:"balance"` + TotalReceived string `json:"totalReceived"` + TotalSent string `json:"totalSent"` + UnconfirmedBalance string `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*Tx `json:"transactions,omitempty"` + Txids []string `json:"transactions,omitempty"` // this is intentional, we return either Transactions or Txids + Page int `json:"page"` + TotalPages int `json:"totalPages"` + TxsOnPage int `json:"txsOnPage"` } diff --git a/api/worker.go b/api/worker.go index afcfb19c..ec5dc49f 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "fmt" "math/big" "time" @@ -33,10 +34,10 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is } // GetTransaction reads transaction data from txid -func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) (*Tx, error) { +func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool) (*Tx, error) { bchainTx, height, err := w.txCache.GetTransaction(txid, bestheight) if err != nil { - return nil, errors.Annotatef(err, "txCache.GetTransaction %v", txid) + return nil, NewApiError(fmt.Sprintf("Tx not found, %v", err), true) } var blockhash string if bchainTx.Confirmations > 0 { @@ -56,20 +57,42 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) vin.ScriptSig.Hex = bchainVin.ScriptSig.Hex // bchainVin.Txid=="" is coinbase transaction if bchainVin.Txid != "" { - otx, _, err := w.txCache.GetTransaction(bchainVin.Txid, bestheight) + // load spending addresses from TxAddresses + ta, err := w.db.GetTxAddresses(bchainVin.Txid) if err != nil { - return nil, errors.Annotatef(err, "txCache.GetTransaction %v", bchainVin.Txid) + return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) } - if len(otx.Vout) > int(vin.Vout) { - vout := &otx.Vout[vin.Vout] - vin.ValueSat = vout.ValueSat - vin.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) - valInSat.Add(&valInSat, &vout.ValueSat) - if vout.Address != nil { - a := vout.Address.String() - vin.Addr = a + if ta == nil { + // mempool transactions are not in TxAddresses, all confirmed should be there, log a problem + if bchainTx.Confirmations > 0 { + glog.Warning("DB inconsistency: tx ", bchainVin.Txid, ": not found in txAddresses") + } + // try to load from backend + otx, _, err := w.txCache.GetTransaction(bchainVin.Txid, bestheight) + if err != nil { + return nil, errors.Annotatef(err, "txCache.GetTransaction %v", bchainVin.Txid) + } + if len(otx.Vout) > int(vin.Vout) { + vout := &otx.Vout[vin.Vout] + vin.ValueSat = vout.ValueSat + if vout.Address != nil { + a := vout.Address.String() + vin.Addr = a + } + } + } else { + if len(ta.Outputs) > int(vin.Vout) { + output := &ta.Outputs[vin.Vout] + vin.ValueSat = output.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + a, _ := output.Addresses(w.chainParser) + if len(a) > 0 { + vin.Addr = a[0] + } } } + vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + valInSat.Add(&valInSat, &vin.ValueSat) } } vouts := make([]Vout, len(bchainTx.Vout)) @@ -82,7 +105,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses - if spendingTx { + if spendingTxs { // TODO } } @@ -100,7 +123,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) Confirmations: bchainTx.Confirmations, Fees: w.chainParser.AmountToDecimalString(&feesSat), Locktime: bchainTx.LockTime, - WithSpends: spendingTx, + WithSpends: spendingTxs, Time: bchainTx.Time, Txid: bchainTx.Txid, ValueIn: w.chainParser.AmountToDecimalString(&valInSat), @@ -112,11 +135,11 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) return r, nil } -func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) { +func (w *Worker) getAddressTxids(address string, mempool bool) ([]string, error) { var err error txids := make([]string, 0) if !mempool { - err = s.db.GetTransactions(address, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { + err = w.db.GetTransactions(address, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { txids = append(txids, txid) return nil }) @@ -124,7 +147,7 @@ func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return nil, err } } else { - m, err := s.chain.GetMempoolTransactions(address) + m, err := w.chain.GetMempoolTransactions(address) if err != nil { return nil, err } @@ -223,9 +246,12 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn // GetAddress computes address value and gets transactions for given address func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids bool) (*Address, error) { start := time.Now() + if page < 0 { + page = 0 + } ba, err := w.db.GetAddressBalance(address) if err != nil { - return nil, errors.Annotatef(err, "GetAddressBalance %v", address) + return nil, NewApiError(fmt.Sprintf("Address not found, %v", err), true) } if ba == nil { return nil, NewApiError("Address not found", true) @@ -235,19 +261,20 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b return nil, errors.Annotatef(err, "getAddressTxids %v false", address) } txc = UniqueTxidsInReverse(txc) - txm, err := w.getAddressTxids(address, true) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v true", address) + var txm []string + // mempool only on the first page + if page == 0 { + txm, err = w.getAddressTxids(address, true) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v true", address) + } + txm = UniqueTxidsInReverse(txm) } - txm = UniqueTxidsInReverse(txm) bestheight, _, err := w.db.GetBestBlock() if err != nil { return nil, errors.Annotatef(err, "GetBestBlock") } // paging - if page < 0 { - page = 0 - } from := page * txsOnPage totalPages := len(txc) / txsOnPage if from >= len(txc) { diff --git a/static/css/main.css b/static/css/main.css index 8265aadd..b52b682d 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -15,6 +15,14 @@ a { text-decoration: none; } +h1 small { + font-size: 65%; +} + +h3 { + margin-top: 20px; +} + .octicon { color: #777; display: inline-block; @@ -175,4 +183,9 @@ a { .alert .data-table { margin: 0; -} \ No newline at end of file +} + +.navbar-nav .nav-link { + padding-right: 0; + padding-left: .25rem; +} diff --git a/static/templates/error.html b/static/templates/error.html index 7ad0ad31..0b75378b 100644 --- a/static/templates/error.html +++ b/static/templates/error.html @@ -1,4 +1,4 @@ {{define "specific"}}

Error

-

{{.Error.Text}}

+

{{.Error.Text}}

{{end}} \ No newline at end of file From be19523065f56747f194c27c7ccc016494248e9d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 29 Aug 2018 00:25:26 +0200 Subject: [PATCH 036/123] Refactor of address handling, renamed addrID to addr descriptor addrDesc --- api/types.go | 28 ++-- api/worker.go | 117 ++++--------- bchain/baseparser.go | 53 ------ bchain/coins/bch/bcashparser.go | 22 +-- bchain/coins/bch/bcashparser_test.go | 31 +--- bchain/coins/btc/bitcoinparser.go | 66 ++++---- bchain/coins/btc/bitcoinparser_test.go | 53 +++--- bchain/coins/dogecoin/dogecoinparser_test.go | 30 +--- bchain/coins/eth/ethparser.go | 27 +-- bchain/coins/eth/ethparser_test.go | 21 +-- bchain/coins/litecoin/litecoinparser_test.go | 30 +--- bchain/coins/monacoin/monacoin_test.go | 32 +--- bchain/coins/monacoin/monacoinparser.go | 6 +- bchain/coins/namecoin/namecoinparser_test.go | 8 +- bchain/coins/vertcoin/vertcoinparser_test.go | 22 +-- bchain/coins/zec/zcashparser.go | 22 ++- bchain/coins/zec/zcashparser_test.go | 18 -- bchain/mempool_nonutxo.go | 30 ++-- bchain/mempool_utxo.go | 35 ++-- bchain/tests/rpc/rpc.go | 1 - bchain/types.go | 18 +- db/rocksdb.go | 166 +++++++++---------- db/rocksdb_test.go | 46 ++--- server/public.go | 6 +- server/socketio.go | 71 +++++--- static/templates/txdetail.html | 15 +- 26 files changed, 393 insertions(+), 581 deletions(-) diff --git a/api/types.go b/api/types.go index e5c734e4..5e465ecf 100644 --- a/api/types.go +++ b/api/types.go @@ -24,23 +24,23 @@ type ScriptSig struct { } type Vin struct { - Txid string `json:"txid"` - Vout uint32 `json:"vout"` - Sequence int64 `json:"sequence,omitempty"` - N int `json:"n"` - ScriptSig ScriptSig `json:"scriptSig"` - Addr string `json:"addr"` - AddrLink bool `json:"-"` - Value string `json:"value"` - ValueSat big.Int `json:"-"` + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Sequence int64 `json:"sequence,omitempty"` + N int `json:"n"` + ScriptSig ScriptSig `json:"scriptSig"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` } type ScriptPubKey struct { - Hex string `json:"hex"` - Asm string `json:"asm,omitempty"` - Addresses []string `json:"addresses"` - AddrsLink []bool `json:"-"` - Type string `json:"type,omitempty"` + Hex string `json:"hex"` + Asm string `json:"asm,omitempty"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Type string `json:"type,omitempty"` } type Vout struct { Value string `json:"value"` diff --git a/api/worker.go b/api/worker.go index ec5dc49f..397b1247 100644 --- a/api/worker.go +++ b/api/worker.go @@ -33,6 +33,14 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is return w, nil } +func (w *Worker) getAddressesFromVout(vout *bchain.Vout) ([]string, bool, error) { + addrDesc, err := w.chainParser.GetAddrDescFromVout(vout) + if err != nil { + return nil, false, err + } + return w.chainParser.GetAddressesFromAddrDesc(addrDesc) +} + // GetTransaction reads transaction data from txid func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool) (*Tx, error) { bchainTx, height, err := w.txCache.GetTransaction(txid, bestheight) @@ -75,9 +83,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] vin.ValueSat = vout.ValueSat - if vout.Address != nil { - a := vout.Address.String() - vin.Addr = a + vin.Addresses, vin.Searchable, err = w.getAddressesFromVout(vout) + if err != nil { + glog.Errorf("getAddressesFromVout error %v, vout %+v", err, vout) } } } else { @@ -85,9 +93,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool output := &ta.Outputs[vin.Vout] vin.ValueSat = output.ValueSat vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) - a, _ := output.Addresses(w.chainParser) - if len(a) > 0 { - vin.Addr = a[0] + vin.Addresses, vin.Searchable, err = output.Addresses(w.chainParser) + if err != nil { + glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) } } } @@ -104,6 +112,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex + vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses if spendingTxs { // TODO @@ -156,11 +165,11 @@ func (w *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return txids, nil } -func (t *Tx) getAddrVoutValue(addrID string) *big.Int { +func (t *Tx) getAddrVoutValue(addr string) *big.Int { var val big.Int for _, vout := range t.Vout { for _, a := range vout.ScriptPubKey.Addresses { - if a == addrID { + if a == addr { val.Add(&val, &vout.ValueSat) } } @@ -168,11 +177,13 @@ func (t *Tx) getAddrVoutValue(addrID string) *big.Int { return &val } -func (t *Tx) getAddrVinValue(addrID string) *big.Int { +func (t *Tx) getAddrVinValue(addr string) *big.Int { var val big.Int for _, vin := range t.Vin { - if vin.Addr == addrID { - val.Add(&val, &vin.ValueSat) + for _, a := range vin.Addresses { + if a == addr { + val.Add(&val, &vin.ValueSat) + } } } return &val @@ -195,6 +206,7 @@ func UniqueTxidsInReverse(txids []string) []string { } func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx { + var err error var valInSat, valOutSat, feesSat big.Int vins := make([]Vin, len(ta.Inputs)) for i := range ta.Inputs { @@ -204,9 +216,9 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn vin.ValueSat = tai.ValueSat vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) valInSat.Add(&valInSat, &vin.ValueSat) - a, err := tai.Addresses(w.chainParser) - if err == nil && len(a) == 1 { - vin.Addr = a[0] + vin.Addresses, vin.Searchable, err = tai.Addresses(w.chainParser) + if err != nil { + glog.Errorf("tai.Addresses error %v, tx %v, input %v", err, txid, i) } } vouts := make([]Vout, len(ta.Outputs)) @@ -217,9 +229,9 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn vout.ValueSat = tao.ValueSat vout.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) valOutSat.Add(&valOutSat, &vout.ValueSat) - a, err := tao.Addresses(w.chainParser) - if err == nil { - vout.ScriptPubKey.Addresses = a + vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = tao.Addresses(w.chainParser) + if err != nil { + glog.Errorf("tai.Addresses error %v, tx %v, output %v", err, txid, i) } } // for coinbase transactions valIn is 0 @@ -343,74 +355,3 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b glog.Info(address, " finished in ", time.Since(start)) return r, nil } - -// GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddressOld(addrID string, page int, txsOnPage int) (*Address, error) { - glog.Info(addrID, " start") - txc, err := w.getAddressTxids(addrID, false) - txc = UniqueTxidsInReverse(txc) - if err != nil { - return nil, err - } - txm, err := w.getAddressTxids(addrID, true) - if err != nil { - return nil, err - } - txm = UniqueTxidsInReverse(txm) - bestheight, _, err := w.db.GetBestBlock() - if err != nil { - return nil, err - } - lc := len(txc) - if lc > txsOnPage { - lc = txsOnPage - } - txs := make([]*Tx, len(txm)+lc) - txi := 0 - var uBalSat, balSat, totRecvSat, totSentSat big.Int - for _, tx := range txm { - tx, err := w.GetTransaction(tx, bestheight, false) - // mempool transaction may fail - if err != nil { - glog.Error("GetTransaction ", tx, ": ", err) - } else { - uBalSat.Sub(tx.getAddrVoutValue(addrID), tx.getAddrVinValue(addrID)) - txs[txi] = tx - txi++ - } - } - if page < 0 { - page = 0 - } - from := page * txsOnPage - if from > len(txc) { - from = 0 - } - to := from + txsOnPage - for i, tx := range txc { - tx, err := w.GetTransaction(tx, bestheight, false) - if err != nil { - return nil, err - } else { - totRecvSat.Add(&totRecvSat, tx.getAddrVoutValue(addrID)) - totSentSat.Add(&totSentSat, tx.getAddrVinValue(addrID)) - if i >= from && i < to { - txs[txi] = tx - txi++ - } - } - } - balSat.Sub(&totRecvSat, &totSentSat) - r := &Address{ - AddrStr: addrID, - Balance: w.chainParser.AmountToDecimalString(&balSat), - TotalReceived: w.chainParser.AmountToDecimalString(&totRecvSat), - TotalSent: w.chainParser.AmountToDecimalString(&totSentSat), - Transactions: txs[:txi], - TxApperances: len(txc), - UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), - UnconfirmedTxApperances: len(txm), - } - glog.Info(addrID, " finished") - return r, nil -} diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 172a8180..48bb752e 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -10,25 +10,12 @@ import ( "github.com/juju/errors" ) -type AddressFactoryFunc func(string) (Address, error) - // BaseParser implements data parsing/handling functionality base for all other parsers type BaseParser struct { - AddressFactory AddressFactoryFunc BlockAddressesToKeep int AmountDecimalPoint int } -// AddressToOutputScript converts address to ScriptPubKey - currently not implemented -func (p *BaseParser) AddressToOutputScript(address string) ([]byte, error) { - return nil, errors.New("AddressToOutputScript: not implemented") -} - -// OutputScriptToAddresses converts ScriptPubKey to addresses - currently not implemented -func (p *BaseParser) OutputScriptToAddresses(script []byte) ([]string, error) { - return nil, errors.New("OutputScriptToAddresses: not implemented") -} - // ParseBlock parses raw block to our Block struct - currently not implemented func (p *BaseParser) ParseBlock(b []byte) (*Block, error) { return nil, errors.New("ParseBlock: not implemented") @@ -100,13 +87,6 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { return nil, err } vout.JsonValue = "" - if len(vout.ScriptPubKey.Addresses) == 1 { - a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) - if err != nil { - return nil, err - } - vout.Address = a - } } return &tx, nil @@ -241,13 +221,6 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { }, ValueSat: vs, } - if len(pto.Addresses) == 1 { - a, err := p.AddressFactory(pto.Addresses[0]) - if err != nil { - return nil, 0, err - } - vout[i].Address = a - } } tx := Tx{ Blocktime: int64(pt.Blocktime), @@ -260,29 +233,3 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } return &tx, pt.Height, nil } - -type baseAddress struct { - addr string -} - -func NewBaseAddress(addr string) (Address, error) { - return &baseAddress{addr: addr}, nil -} - -func (a baseAddress) String() string { - return a.addr -} - -func (a baseAddress) AreEqual(addr string) bool { - return a.String() == addr -} - -func (a baseAddress) InSlice(addrs []string) bool { - ea := a.String() - for _, addr := range addrs { - if ea == addr { - return true - } - } - return false -} diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 02286ca6..ccfebc29 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -47,15 +47,14 @@ func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) (*BCashParser p := &BCashParser{ BitcoinParser: &btc.BitcoinParser{ BaseParser: &bchain.BaseParser{ - AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, format) }, BlockAddressesToKeep: c.BlockAddressesToKeep, AmountDecimalPoint: 8, }, Params: params, - OutputScriptToAddressesFunc: outputScriptToAddresses, }, AddressFormat: format, } + p.OutputScriptToAddressesFunc = p.outputScriptToAddresses return p, nil } @@ -79,13 +78,13 @@ func GetChainParams(chain string) *chaincfg.Params { return params } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *BCashParser) GetAddrIDFromAddress(address string) ([]byte, error) { - return p.AddressToOutputScript(address) +// GetAddrDescFromAddress returns internal address representation of given address +func (p *BCashParser) GetAddrDescFromAddress(address string) ([]byte, error) { + return p.addressToOutputScript(address) } -// AddressToOutputScript converts bitcoin address to ScriptPubKey -func (p *BCashParser) AddressToOutputScript(address string) ([]byte, error) { +// addressToOutputScript converts bitcoin address to ScriptPubKey +func (p *BCashParser) addressToOutputScript(address string) ([]byte, error) { if isCashAddr(address) { da, err := bchutil.DecodeAddress(address, p.Params) if err != nil { @@ -124,12 +123,13 @@ func isCashAddr(addr string) bool { } // outputScriptToAddresses converts ScriptPubKey to bitcoin addresses -func outputScriptToAddresses(script []byte, params *chaincfg.Params) ([]string, error) { - a, err := bchutil.ExtractPkScriptAddrs(script, params) +func (p *BCashParser) outputScriptToAddresses(script []byte) ([]string, bool, error) { + a, err := bchutil.ExtractPkScriptAddrs(script, p.Params) if err != nil { - return nil, err + return nil, false, err } - return []string{a.EncodeAddress()}, nil + addr := a.EncodeAddress() + return []string{addr}, len(addr) > 0, nil } type bcashAddress struct { diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index d6040b82..ef2867fe 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -91,7 +91,7 @@ func TestBcashAddressInSlice(t *testing.T) { } } -func TestAddressToOutputScript(t *testing.T) { +func Test_GetAddrDescFromAddress(t *testing.T) { parser, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) if err != nil { t.Errorf("NewBCashParser() error = %v", err) @@ -101,21 +101,21 @@ func TestAddressToOutputScript(t *testing.T) { if err != nil { panic(err) } - got1, err := parser.AddressToOutputScript("mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr") + got1, err := parser.GetAddrDescFromAddress("mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr") if err != nil { - t.Errorf("AddressToOutputScript() error = %v", err) + t.Errorf("GetAddrDescFromAddress() error = %v", err) return } if !bytes.Equal(got1, want) { - t.Errorf("AddressToOutputScript() got1 = %v, want %v", got1, want) + t.Errorf("GetAddrDescFromAddress() got1 = %v, want %v", got1, want) } - got2, err := parser.AddressToOutputScript("bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm") + got2, err := parser.GetAddrDescFromAddress("bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm") if err != nil { - t.Errorf("AddressToOutputScript() error = %v", err) + t.Errorf("GetAddrDescFromAddress() error = %v", err) return } if !bytes.Equal(got2, want) { - t.Errorf("AddressToOutputScript() got2 = %v, want %v", got2, want) + t.Errorf("GetAddrDescFromAddress() got2 = %v, want %v", got2, want) } } @@ -127,20 +127,6 @@ var ( ) func init() { - var ( - addr1, addr2, addr3 bchain.Address - err error - ) - addr1, err = newBCashAddress("3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK", Legacy) - if err == nil { - addr2, err = newBCashAddress("2NByHN6A8QYkBATzxf4pRGbCSHD5CEN2TRu", Legacy) - } - if err == nil { - addr3, err = newBCashAddress("2MvZguYaGjM7JihBgNqgLF2Ca2Enb76Hj9D", Legacy) - } - if err != nil { - panic(err) - } testTx1 = bchain.Tx{ Hex: "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700", @@ -168,7 +154,6 @@ func init() { "bitcoincash:pps5f4tu3tl5sjfvnhaeznsjpvst44eddugfcnqpy9", }, }, - Address: addr1, }, }, } @@ -199,7 +184,6 @@ func init() { "bchtest:prxkdrtcrm8xqrh6fvjqfhy3l5nt3w9wmq9fmsvkmz", }, }, - Address: addr2, }, { ValueSat: *big.NewInt(920081157), @@ -210,7 +194,6 @@ func init() { "bchtest:pqjxv4dah42v0erh6r4zxa0gdcxm9w8cpg0qw8tqf6", }, }, - Address: addr3, }, }, } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 6e2f4c69..f741adf9 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -16,7 +16,7 @@ import ( ) // OutputScriptToAddressesFunc converts ScriptPubKey to bitcoin addresses -type OutputScriptToAddressesFunc func(script []byte, params *chaincfg.Params) ([]string, error) +type OutputScriptToAddressesFunc func(script []byte) ([]string, bool, error) // BitcoinParser handle type BitcoinParser struct { @@ -27,15 +27,15 @@ type BitcoinParser struct { // NewBitcoinParser returns new BitcoinParser instance func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser { - return &BitcoinParser{ - &bchain.BaseParser{ - AddressFactory: bchain.NewBaseAddress, + p := &BitcoinParser{ + BaseParser: &bchain.BaseParser{ BlockAddressesToKeep: c.BlockAddressesToKeep, AmountDecimalPoint: 8, }, - params, - outputScriptToAddresses, + Params: params, } + p.OutputScriptToAddressesFunc = p.outputScriptToAddresses + return p } // GetChainParams contains network parameters for the main Bitcoin network, @@ -51,18 +51,28 @@ func GetChainParams(chain string) *chaincfg.Params { return &chaincfg.MainNetParams } -// GetAddrIDFromVout returns internal address representation of given transaction output -func (p *BitcoinParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { +// GetAddrDescFromVout returns internal address representation (descriptor) of given transaction output +func (p *BitcoinParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { return hex.DecodeString(output.ScriptPubKey.Hex) } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *BitcoinParser) GetAddrIDFromAddress(address string) ([]byte, error) { - return p.AddressToOutputScript(address) +// GetAddrDescFromAddress returns internal address representation (descriptor) of given address +func (p *BitcoinParser) GetAddrDescFromAddress(address string) ([]byte, error) { + return p.addressToOutputScript(address) } -// AddressToOutputScript converts bitcoin address to ScriptPubKey -func (p *BitcoinParser) AddressToOutputScript(address string) ([]byte, error) { +// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable +func (p *BitcoinParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { + return p.outputScriptToAddresses(addrDesc) +} + +// GetScriptFromAddrDesc returns output script for given address descriptor +func (p *BitcoinParser) GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) { + return addrDesc, nil +} + +// addressToOutputScript converts bitcoin address to ScriptPubKey +func (p *BitcoinParser) addressToOutputScript(address string) ([]byte, error) { da, err := btcutil.DecodeAddress(address, p.Params) if err != nil { return nil, err @@ -74,22 +84,21 @@ func (p *BitcoinParser) AddressToOutputScript(address string) ([]byte, error) { return script, nil } -// OutputScriptToAddresses converts ScriptPubKey to bitcoin addresses -func (p *BitcoinParser) OutputScriptToAddresses(script []byte) ([]string, error) { - return p.OutputScriptToAddressesFunc(script, p.Params) -} - // outputScriptToAddresses converts ScriptPubKey to bitcoin addresses -func outputScriptToAddresses(script []byte, params *chaincfg.Params) ([]string, error) { - _, addresses, _, err := txscript.ExtractPkScriptAddrs(script, params) +func (p *BitcoinParser) outputScriptToAddresses(script []byte) ([]string, bool, error) { + sc, addresses, _, err := txscript.ExtractPkScriptAddrs(script, p.Params) if err != nil { - return nil, err + return nil, false, err } rv := make([]string, len(addresses)) for i, a := range addresses { rv[i] = a.EncodeAddress() } - return rv, nil + var s bool + if sc != txscript.NonStandardTy && sc != txscript.NullDataTy { + s = true + } + return rv, s, nil } func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.Tx { @@ -117,7 +126,7 @@ func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.T for i, out := range t.TxOut { addrs := []string{} if parseAddresses { - addrs, _ = p.OutputScriptToAddresses(out.PkScript) + addrs, _, _ = p.OutputScriptToAddressesFunc(out.PkScript) } s := bchain.ScriptPubKey{ Hex: hex.EncodeToString(out.PkScript), @@ -156,17 +165,6 @@ func (p *BitcoinParser) ParseTx(b []byte) (*bchain.Tx, error) { } tx := p.TxFromMsgTx(&t, true) tx.Hex = hex.EncodeToString(b) - - for i, vout := range tx.Vout { - if len(vout.ScriptPubKey.Addresses) == 1 { - a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) - if err != nil { - return nil, err - } - tx.Vout[i].Address = a - } - } - return &tx, nil } diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 79d02816..4b4308a1 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -1,4 +1,4 @@ -// +build unittest +// build unittest package btc @@ -10,7 +10,7 @@ import ( "testing" ) -func TestAddressToOutputScript(t *testing.T) { +func Test_GetAddrDescFromAddress(t *testing.T) { type args struct { address string } @@ -49,20 +49,20 @@ func TestAddressToOutputScript(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -func TestOutputScriptToAddresses(t *testing.T) { +func Test_GetAddressesFromAddrDesc(t *testing.T) { type args struct { script string } @@ -70,43 +70,62 @@ func TestOutputScriptToAddresses(t *testing.T) { name string args args want []string + want2 bool wantErr bool }{ { name: "P2PKH", args: args{script: "76a914be027bf3eac907bd4ac8cb9c5293b6f37662722088ac"}, want: []string{"1JKgN43B9SyLuZH19H5ECvr4KcfrbVHzZ6"}, + want2: true, wantErr: false, }, { name: "P2SH", args: args{script: "a9140394b3cf9a44782c10105b93962daa8dba304d7f87"}, want: []string{"321x69Cb9HZLWwAWGiUBT1U81r1zPLnEjL"}, + want2: true, wantErr: false, }, { name: "P2WPKH", args: args{script: "00141c12afc6b2602607fdbc209f2a053c54ecd2c673"}, want: []string{"bc1qrsf2l34jvqnq0lduyz0j5pfu2nkd93nnq0qggn"}, + want2: true, wantErr: false, }, { name: "P2WSH", args: args{script: "002003973a40ec94c0d10f6f6f0e7a62ba2044b7d19db6ff2bf60651e17fb29d8d29"}, want: []string{"bc1qqwtn5s8vjnqdzrm0du885c46ypzt05vakmljhasx28shlv5a355sw5exgr"}, + want2: true, + wantErr: false, + }, + { + // TODO handle OP_RETURN better + name: "OP_RETURN", + args: args{script: "6a0461686f6a"}, + want: []string{}, + want2: false, wantErr: false, }, } + + parser := NewBitcoinParser(GetChainParams("main"), &Configuration{}) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b, _ := hex.DecodeString(tt.args.script) - got, err := outputScriptToAddresses(b, GetChainParams("main")) + got, got2, err := parser.GetAddressesFromAddrDesc(b) if (err != nil) != tt.wantErr { t.Errorf("outputScriptToAddresses() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("outputScriptToAddresses() = %v, want %v", got, tt.want) + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) } }) } @@ -120,21 +139,6 @@ var ( ) func init() { - var ( - addr1, addr2, addr3 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK") - if err == nil { - addr2, err = bchain.NewBaseAddress("2NByHN6A8QYkBATzxf4pRGbCSHD5CEN2TRu") - } - if err == nil { - addr3, err = bchain.NewBaseAddress("2MvZguYaGjM7JihBgNqgLF2Ca2Enb76Hj9D") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700", Blocktime: 1519053802, @@ -161,7 +165,6 @@ func init() { "3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK", }, }, - Address: addr1, }, }, } @@ -192,7 +195,6 @@ func init() { "2NByHN6A8QYkBATzxf4pRGbCSHD5CEN2TRu", }, }, - Address: addr2, }, { ValueSat: *big.NewInt(920081157), @@ -203,7 +205,6 @@ func init() { "2MvZguYaGjM7JihBgNqgLF2Ca2Enb76Hj9D", }, }, - Address: addr3, }, }, } diff --git a/bchain/coins/dogecoin/dogecoinparser_test.go b/bchain/coins/dogecoin/dogecoinparser_test.go index 0b736c51..f9c1037e 100644 --- a/bchain/coins/dogecoin/dogecoinparser_test.go +++ b/bchain/coins/dogecoin/dogecoinparser_test.go @@ -15,7 +15,7 @@ import ( "testing" ) -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -54,14 +54,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -76,24 +76,6 @@ var ( ) func init() { - var ( - addr1, addr2, addr3, addr4 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("DSvXNiqvG42wdteLqh3i6inxgDTs8Y9w2i") - if err == nil { - addr2, err = bchain.NewBaseAddress("DRemF3ZcqJ1PFeM7e7sXzzwQJKR8GNUtwK") - } - if err == nil { - addr3, err = bchain.NewBaseAddress("DJa8bWDrZKu4HgsYRYWuJrvxt6iTYuvXJ6") - } - if err == nil { - addr4, err = bchain.NewBaseAddress("DDTtqnuZ5kfRT5qh2c7sNtqrJmV3iXYdGG") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "01000000016b3c0c53267964120acf7f7e72217e3f463e52ce622f89659f6a6bb8e69a4d91000000006c493046022100a96454237e3a020994534583e28c04757881374bceac89f933ea9ff00b4db259022100fbb757ff7ea4f02c4e42556b2834c61eba1f1af605db089d836a0614d90a3b46012103cebdde6d1046e285df4f48497bc50dc20a4a258ca5b7308cb0a929c9fdadcd9dffffffff0217e823ca7f0200001976a914eef21768a546590993e313c7f3dfadf6a6efa1e888acaddf4cba010000001976a914e0fee2ea29dd9c6c759d8341bd0da4c4f738cced88ac00000000", Blocktime: 1519053456, @@ -120,7 +102,6 @@ func init() { "DSvXNiqvG42wdteLqh3i6inxgDTs8Y9w2i", }, }, - Address: addr1, }, { ValueSat: *big.NewInt(7420567469), @@ -131,7 +112,6 @@ func init() { "DRemF3ZcqJ1PFeM7e7sXzzwQJKR8GNUtwK", }, }, - Address: addr2, }, }, } @@ -162,7 +142,6 @@ func init() { "DJa8bWDrZKu4HgsYRYWuJrvxt6iTYuvXJ6", }, }, - Address: addr3, }, { ValueSat: *big.NewInt(999999890000000), @@ -173,7 +152,6 @@ func init() { "DDTtqnuZ5kfRT5qh2c7sNtqrJmV3iXYdGG", }, }, - Address: addr4, }, }, } diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index 82d08153..f933b761 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -23,7 +23,6 @@ type EthereumParser struct { // NewEthereumParser returns new EthereumParser instance func NewEthereumParser() *EthereumParser { return &EthereumParser{&bchain.BaseParser{ - AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: 0, AmountDecimalPoint: 18, }} @@ -68,7 +67,6 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma txid := ethHashToHash(tx.Hash) var ( fa, ta []string - addr bchain.Address err error ) if len(tx.From) > 2 { @@ -76,10 +74,6 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } if len(tx.To) > 2 { ta = []string{tx.To} - addr, err = p.AddressFactory(tx.To) - if err != nil { - return nil, err - } } // temporarily, the complete rpcTransaction without BlockHash is marshalled and hex encoded to bchain.Tx.Hex bh := tx.BlockHash @@ -119,26 +113,25 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma // Hex Addresses: ta, }, - Address: addr, }, }, }, nil } -// GetAddrIDFromVout returns internal address representation of given transaction output -func (p *EthereumParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { +// GetAddrDescFromVout returns internal address representation of given transaction output +func (p *EthereumParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { if len(output.ScriptPubKey.Addresses) != 1 { return nil, bchain.ErrAddressMissing } - return p.GetAddrIDFromAddress(output.ScriptPubKey.Addresses[0]) + return p.GetAddrDescFromAddress(output.ScriptPubKey.Addresses[0]) } func has0xPrefix(s string) bool { return len(s) >= 2 && s[0] == '0' && (s[1]|32) == 'x' } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *EthereumParser) GetAddrIDFromAddress(address string) ([]byte, error) { +// GetAddrDescFromAddress returns internal address representation of given address +func (p *EthereumParser) GetAddrDescFromAddress(address string) ([]byte, error) { // github.com/ethereum/go-ethereum/common.HexToAddress does not handle address errors, using own decoding if has0xPrefix(address) { address = address[2:] @@ -152,6 +145,16 @@ func (p *EthereumParser) GetAddrIDFromAddress(address string) ([]byte, error) { return hex.DecodeString(address) } +// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable +func (p *EthereumParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { + return []string{hexutil.Encode(addrDesc)}, true, nil +} + +// GetScriptFromAddrDesc returns output script for given address descriptor +func (p *EthereumParser) GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) { + return addrDesc, nil +} + func hexDecode(s string) ([]byte, error) { b, err := hexutil.Decode(s) if err != nil && err != hexutil.ErrEmptyString { diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index 40069a55..2a7ee8bf 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -10,7 +10,7 @@ import ( "testing" ) -func TestEthParser_GetAddrIDFromAddress(t *testing.T) { +func TestEthParser_GetAddrDescFromAddress(t *testing.T) { type args struct { address string } @@ -51,14 +51,14 @@ func TestEthParser_GetAddrIDFromAddress(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := NewEthereumParser() - got, err := p.GetAddrIDFromAddress(tt.args.address) + got, err := p.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("EthParser.GetAddrIDFromAddress() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("EthParser.GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("EthParser.GetAddrIDFromAddress() = %v, want %v", h, tt.want) + t.Errorf("EthParser.GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -71,17 +71,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") - if err == nil { - addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") - } - if err != nil { - panic(err) - } testTx1 = bchain.Tx{ Blocktime: 1521515026, @@ -99,7 +88,6 @@ func init() { ScriptPubKey: bchain.ScriptPubKey{ Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, }, - Address: addr1, }, }, } @@ -120,7 +108,6 @@ func init() { ScriptPubKey: bchain.ScriptPubKey{ Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, }, - Address: addr2, }, }, } diff --git a/bchain/coins/litecoin/litecoinparser_test.go b/bchain/coins/litecoin/litecoinparser_test.go index a76faf63..1e046bcd 100644 --- a/bchain/coins/litecoin/litecoinparser_test.go +++ b/bchain/coins/litecoin/litecoinparser_test.go @@ -11,7 +11,7 @@ import ( "testing" ) -func TestAddressToOutputScript_Testnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Testnet(t *testing.T) { type args struct { address string } @@ -44,20 +44,20 @@ func TestAddressToOutputScript_Testnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -108,14 +108,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -128,18 +128,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("LMgENNXzzuPxp7vfMjDrCU44bsmrEMgqvc") - if err == nil { - addr2, err = bchain.NewBaseAddress("LV1ByjbJNFTHyFQqwqwdJXKJznYDzXzg4B") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "02000000031e1977dc524bec5929e95d8d0946812944b7b5bda12f5b99fdf557773f2ee65e0100000000ffffffff8a398e44546dce0245452b90130e86832b21fd68f26662bc33aeb7c6c115d23c1900000000ffffffffb807ab93a7fcdff7af6d24581a4a18aa7c1db1ebecba2617a6805b009513940f0c00000000ffffffff020001a04a000000001976a9141ae882e788091732da6910595314447c9e38bd8d88ac27440f00000000001976a9146b474cbf0f6004329b630bdd4798f2c23d1751b688ac00000000", Blocktime: 1519053456, @@ -182,7 +170,6 @@ func init() { "LMgENNXzzuPxp7vfMjDrCU44bsmrEMgqvc", }, }, - Address: addr1, }, { ValueSat: *big.NewInt(1000487), @@ -193,7 +180,6 @@ func init() { "LV1ByjbJNFTHyFQqwqwdJXKJznYDzXzg4B", }, }, - Address: addr2, }, }, } diff --git a/bchain/coins/monacoin/monacoin_test.go b/bchain/coins/monacoin/monacoin_test.go index 6e41db33..9f559749 100644 --- a/bchain/coins/monacoin/monacoin_test.go +++ b/bchain/coins/monacoin/monacoin_test.go @@ -1,4 +1,4 @@ -// build unittest +// +build unittest package monacoin @@ -11,7 +11,7 @@ import ( "testing" ) -func TestAddressToOutputScript_Testnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Testnet(t *testing.T) { type args struct { address string } @@ -44,20 +44,20 @@ func TestAddressToOutputScript_Testnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -108,14 +108,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -128,18 +128,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("MWpWpANNQRskQHcuY5ZQpN4BVynQxmSxRb") - if err == nil { - addr2, err = bchain.NewBaseAddress("MGtFpCVyKEHNtpVNesxPMxYuQayoEBX5yZ") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "0200000003e44ef4e5fe2e4345f1e1340afe396c780773e3834a5bffb153a2faf510e2845e000000006a47304402205ebd735621eaaf512441998727a37e99be94e5ecded54601ea3eebac9282bc2502207d48da44e1c883579c6cd8c2b8ccfb5380e5ac71affe70b475d2b558e0f7bd4b01210391f72b34c04855ce16b97dd79b0ba78fc4b26f40abce853c33788e348cb79c3bfeffffff0ad690a74c43c0df9527c516d26e31fa47e15471a2ead65757b672522888e920010000006b48304502210091a473124bf506edbb095951aa1a32c76bea7eba4020ae2858314961b1a83de602205c3818e517cf830a95a1208fc84aa343faaeeaaa96eab76238379769598ab2d40121038c217e5de8e375ed6cf648e96ec6bfb9e0fbcf5ae3945a5ea60d16919d9c8b68feffffffb9aa4aed4ad4c4b95419e132a43db34aa03a7ec35ef0beecdd627f9ca07bda03010000006a47304402204906d973ac9b4786403f8f8fc2b2ad2e6745ea01a93336b4b67af1d7d1b625cc022016820be905ffd6e11949da79e7a1c7eb97939421a04e0645c8caef8fc585f7ca012102b5f647c4eb677e952913c0b6934c12b29dc50afba8b558b1677ffd2d78c84a88feffffff02f6da4601000000001976a914fb69fe6dcfe88557dc0ce0ea65bd7cf02f5e4f5b88ac8bfd8c57000000001976a914628d603ac50d656e3311ff0cd5490b4c5cdd92ea88ac25fd1400", Blocktime: 1530902705, @@ -182,7 +170,6 @@ func init() { "MWpWpANNQRskQHcuY5ZQpN4BVynQxmSxRb", }, }, - Address: addr1, }, { ValueSat: *big.NewInt(1468857739), @@ -193,7 +180,6 @@ func init() { "MGtFpCVyKEHNtpVNesxPMxYuQayoEBX5yZ", }, }, - Address: addr2, }, }, } diff --git a/bchain/coins/monacoin/monacoinparser.go b/bchain/coins/monacoin/monacoinparser.go index 1268e843..88d0870c 100644 --- a/bchain/coins/monacoin/monacoinparser.go +++ b/bchain/coins/monacoin/monacoinparser.go @@ -91,11 +91,11 @@ func GetMonaChainParams(chain string) *monacoinCfg.Params { // GetAddrIDFromAddress returns internal address representation of given address func (p *MonacoinParser) GetAddrIDFromAddress(address string) ([]byte, error) { - return p.AddressToOutputScript(address) + return p.addressToOutputScript(address) } -// AddressToOutputScript converts monacoin address to ScriptPubKey -func (p *MonacoinParser) AddressToOutputScript(address string) ([]byte, error) { +// addressToOutputScript converts monacoin address to ScriptPubKey +func (p *MonacoinParser) addressToOutputScript(address string) ([]byte, error) { switch p.Params.Net { case MainnetMagic: da, err := monautil.DecodeAddress(address, &MonaMainParams) diff --git a/bchain/coins/namecoin/namecoinparser_test.go b/bchain/coins/namecoin/namecoinparser_test.go index 0f22015d..dcf34b5f 100644 --- a/bchain/coins/namecoin/namecoinparser_test.go +++ b/bchain/coins/namecoin/namecoinparser_test.go @@ -13,7 +13,7 @@ import ( "testing" ) -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -40,14 +40,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } diff --git a/bchain/coins/vertcoin/vertcoinparser_test.go b/bchain/coins/vertcoin/vertcoinparser_test.go index 0cbf01d5..5cdcb5c6 100644 --- a/bchain/coins/vertcoin/vertcoinparser_test.go +++ b/bchain/coins/vertcoin/vertcoinparser_test.go @@ -11,7 +11,7 @@ import ( "testing" ) -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -56,14 +56,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -76,18 +76,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("Vp1UqzsmVecaexfbWFGSFFL5x1g2XQnrGR") - if err == nil { - addr2, err = bchain.NewBaseAddress("38A1RNvbA5c9wNRfyLVn1FCH5TPKJVG8YR") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "010000000146fd781834a34e0399ccda1edf9ec47d715e17d904ad0958d533a240b3605ad6000000006a473044022026b352a0c35c232342339e2b50ec9f04587b990d5213174e368cc76dc82686f002207d0787461ad846825872a50d3d6fc748d5a836575c1daf6ad0ca602f9c4a8826012103d36b6b829c571ed7caa565eca9bdc2aa36519b7ab8551ace5edb0356d477ad3cfdffffff020882a400000000001976a91499b16da88a7e29b913b6131df2644d6d06cb331b88ac80f0fa020000000017a91446eb90e002f137f05385896c882fe000cc2e967f8774870e00", Blocktime: 1529925180, @@ -114,7 +102,6 @@ func init() { "Vp1UqzsmVecaexfbWFGSFFL5x1g2XQnrGR", }, }, - Address: addr1, }, { ValueSat: *big.NewInt(50000000), @@ -125,7 +112,6 @@ func init() { "38A1RNvbA5c9wNRfyLVn1FCH5TPKJVG8YR", }, }, - Address: addr2, }, }, } diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index dc2d84d9..84f422db 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -7,6 +7,7 @@ import ( "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/juju/errors" ) const ( @@ -24,7 +25,6 @@ type ZCashParser struct { func NewZCashParser(c *btc.Configuration) *ZCashParser { return &ZCashParser{ &bchain.BaseParser{ - AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, AmountDecimalPoint: 8, }, @@ -51,8 +51,8 @@ func GetChainParams(chain string) *chaincfg.Params { return params } -// GetAddrIDFromVout returns internal address representation of given transaction output -func (p *ZCashParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { +// GetAddrDescFromVout returns internal address representation of given transaction output +func (p *ZCashParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { if len(output.ScriptPubKey.Addresses) != 1 { return nil, nil } @@ -60,8 +60,20 @@ func (p *ZCashParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { return hash, err } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *ZCashParser) GetAddrIDFromAddress(address string) ([]byte, error) { +// GetAddrDescFromAddress returns internal address representation of given address +func (p *ZCashParser) GetAddrDescFromAddress(address string) ([]byte, error) { hash, _, err := utils.CheckDecode(address) return hash, err } + +// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable +func (p *ZCashParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { + // TODO implement + return nil, false, errors.New("GetAddressesFromAddrDesc: not implemented") +} + +// GetScriptFromAddrDesc returns output script for given address descriptor +func (p *ZCashParser) GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) { + // TODO implement + return nil, errors.New("GetScriptFromAddrDesc: not implemented") +} diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index b1bfd773..c5e02b5c 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -19,21 +19,6 @@ var ( ) func init() { - var ( - addr1, addr2, addr3 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("t1Y4yL14ACHaAbjemkdpW7nYNHWnv1yQbDA") - if err == nil { - addr2, err = bchain.NewBaseAddress("t1VmHTTwpEtwvojxodN2CSQqLYi1hzY3cAq") - } - if err == nil { - addr3, err = bchain.NewBaseAddress("t1ecxMXpphUTRQXGLXnVhJ6ucqD3DZipddg") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "02000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e1802", Blocktime: 1521645728, @@ -60,7 +45,6 @@ func init() { "t1Y4yL14ACHaAbjemkdpW7nYNHWnv1yQbDA", }, }, - Address: addr1, }, }, } @@ -91,7 +75,6 @@ func init() { "t1VmHTTwpEtwvojxodN2CSQqLYi1hzY3cAq", }, }, - Address: addr2, }, { ValueSat: *big.NewInt(10000000), @@ -102,7 +85,6 @@ func init() { "t1ecxMXpphUTRQXGLXnVhJ6ucqD3DZipddg", }, }, - Address: addr3, }, }, } diff --git a/bchain/mempool_nonutxo.go b/bchain/mempool_nonutxo.go index 8ccf018c..3d1aa66f 100644 --- a/bchain/mempool_nonutxo.go +++ b/bchain/mempool_nonutxo.go @@ -12,7 +12,7 @@ type NonUTXOMempool struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrIDToTx map[string][]outpoint + addrDescToTx map[string][]outpoint } // NewNonUTXOMempool creates new mempool handler. @@ -23,13 +23,13 @@ func NewNonUTXOMempool(chain BlockChain) *NonUTXOMempool { // GetTransactions returns slice of mempool transactions for given address func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { parser := m.chain.GetChainParser() - addrID, err := parser.GetAddrIDFromAddress(address) + addrDesc, err := parser.GetAddrDescFromAddress(address) if err != nil { return nil, err } m.mux.Lock() defer m.mux.Unlock() - outpoints := m.addrIDToTx[string(addrID)] + outpoints := m.addrDescToTx[string(addrDesc)] txs := make([]string, 0, len(outpoints)) for _, o := range outpoints { txs = append(txs, o.txid) @@ -37,11 +37,11 @@ func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { return txs, nil } -func (m *NonUTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrIDToTx map[string][]outpoint) { +func (m *NonUTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]outpoint) { m.mux.Lock() defer m.mux.Unlock() m.txToInputOutput = newTxToInputOutput - m.addrIDToTx = newAddrIDToTx + m.addrDescToTx = newAddrDescToTx } // Resync gets mempool transactions and maps outputs to transactions. @@ -57,7 +57,7 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int parser := m.chain.GetChainParser() // allocate slightly larger capacity of the maps newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) - newAddrIDToTx := make(map[string][]outpoint, len(m.addrIDToTx)+5) + newAddrDescToTx := make(map[string][]outpoint, len(m.addrDescToTx)+5) for _, txid := range txs { io, exists := m.txToInputOutput[txid] if !exists { @@ -68,15 +68,15 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int } io = make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) for _, output := range tx.Vout { - addrID, err := parser.GetAddrIDFromVout(&output) + addrDesc, err := parser.GetAddrDescFromVout(&output) if err != nil { if err != ErrAddressMissing { - glog.Error("error in output addrID in ", txid, " ", output.N, ": ", err) + glog.Error("error in output addrDesc in ", txid, " ", output.N, ": ", err) } continue } - if len(addrID) > 0 { - io = append(io, addrIndex{string(addrID), int32(output.N)}) + if len(addrDesc) > 0 { + io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0]) @@ -85,22 +85,22 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int for _, input := range tx.Vin { for i, a := range input.Addresses { if len(a) > 0 { - addrID, err := parser.GetAddrIDFromAddress(a) + addrDesc, err := parser.GetAddrDescFromAddress(a) if err != nil { - glog.Error("error in input addrID in ", txid, " ", a, ": ", err) + glog.Error("error in input addrDesc in ", txid, " ", a, ": ", err) continue } - io = append(io, addrIndex{string(addrID), int32(^i)}) + io = append(io, addrIndex{string(addrDesc), int32(^i)}) } } } } newTxToInputOutput[txid] = io for _, si := range io { - newAddrIDToTx[si.addrID] = append(newAddrIDToTx[si.addrID], outpoint{txid, si.n}) + newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], outpoint{txid, si.n}) } } - m.updateMappings(newTxToInputOutput, newAddrIDToTx) + m.updateMappings(newTxToInputOutput, newAddrDescToTx) glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txToInputOutput), " transactions in mempool") return len(m.txToInputOutput), nil } diff --git a/bchain/mempool_utxo.go b/bchain/mempool_utxo.go index 1963bc19..f89daa29 100644 --- a/bchain/mempool_utxo.go +++ b/bchain/mempool_utxo.go @@ -9,8 +9,8 @@ import ( // addrIndex and outpoint are used also in non utxo mempool type addrIndex struct { - addrID string - n int32 + addrDesc string + n int32 } type outpoint struct { @@ -28,7 +28,7 @@ type UTXOMempool struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrIDToTx map[string][]outpoint + addrDescToTx map[string][]outpoint chanTxid chan string chanAddrIndex chan txidio onNewTxAddr func(txid string, addr string) @@ -70,13 +70,13 @@ func NewUTXOMempool(chain BlockChain, workers int, subworkers int) *UTXOMempool // GetTransactions returns slice of mempool transactions for given address func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { parser := m.chain.GetChainParser() - addrID, err := parser.GetAddrIDFromAddress(address) + addrDesc, err := parser.GetAddrDescFromAddress(address) if err != nil { return nil, err } m.mux.Lock() defer m.mux.Unlock() - outpoints := m.addrIDToTx[string(addrID)] + outpoints := m.addrDescToTx[string(addrDesc)] txs := make([]string, 0, len(outpoints)) for _, o := range outpoints { txs = append(txs, o.txid) @@ -84,15 +84,14 @@ func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { return txs, nil } -func (m *UTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrIDToTx map[string][]outpoint) { +func (m *UTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]outpoint) { m.mux.Lock() defer m.mux.Unlock() m.txToInputOutput = newTxToInputOutput - m.addrIDToTx = newAddrIDToTx + m.addrDescToTx = newAddrDescToTx } func (m *UTXOMempool) getInputAddress(input outpoint) *addrIndex { - // TODO - possibly get from DB unspenttxs - however some output txs can be also in mempool itx, err := m.chain.GetTransactionForMempool(input.txid) if err != nil { glog.Error("cannot get transaction ", input.txid, ": ", err) @@ -102,12 +101,12 @@ func (m *UTXOMempool) getInputAddress(input outpoint) *addrIndex { glog.Error("Vout len in transaction ", input.txid, " ", len(itx.Vout), " input.Vout=", input.vout) return nil } - addrID, err := m.chain.GetChainParser().GetAddrIDFromVout(&itx.Vout[input.vout]) + addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&itx.Vout[input.vout]) if err != nil { - glog.Error("error in addrID in ", input.txid, " ", input.vout, ": ", err) + glog.Error("error in addrDesc in ", input.txid, " ", input.vout, ": ", err) return nil } - return &addrIndex{string(addrID), ^input.vout} + return &addrIndex{string(addrDesc), ^input.vout} } @@ -120,13 +119,13 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul glog.V(2).Info("mempool: gettxaddrs ", txid, ", ", len(tx.Vin), " inputs") io := make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) for _, output := range tx.Vout { - addrID, err := m.chain.GetChainParser().GetAddrIDFromVout(&output) + addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&output) if err != nil { - glog.Error("error in addrID in ", txid, " ", output.N, ": ", err) + glog.Error("error in addrDesc in ", txid, " ", output.N, ": ", err) continue } - if len(addrID) > 0 { - io = append(io, addrIndex{string(addrID), int32(output.N)}) + if len(addrDesc) > 0 { + io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if m.onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { m.onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0]) @@ -177,13 +176,13 @@ func (m *UTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int, e glog.V(2).Info("mempool: resync ", len(txs), " txs") // allocate slightly larger capacity of the maps newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) - newAddrIDToTx := make(map[string][]outpoint, len(m.addrIDToTx)+5) + newAddrDescToTx := make(map[string][]outpoint, len(m.addrDescToTx)+5) dispatched := 0 onNewData := func(txid string, io []addrIndex) { if len(io) > 0 { newTxToInputOutput[txid] = io for _, si := range io { - newAddrIDToTx[si.addrID] = append(newAddrIDToTx[si.addrID], outpoint{txid, si.n}) + newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], outpoint{txid, si.n}) } } } @@ -212,7 +211,7 @@ func (m *UTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int, e tio := <-m.chanAddrIndex onNewData(tio.txid, tio.io) } - m.updateMappings(newTxToInputOutput, newAddrIDToTx) + m.updateMappings(newTxToInputOutput, newAddrDescToTx) m.onNewTxAddr = nil glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txToInputOutput), " transactions in mempool") return len(m.txToInputOutput), nil diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go index 20483573..bc7960a8 100644 --- a/bchain/tests/rpc/rpc.go +++ b/bchain/tests/rpc/rpc.go @@ -96,7 +96,6 @@ func setTxAddresses(parser bchain.BlockChainParser, tx *bchain.Tx) error { if err == nil { for i := 0; i < len(tx.Vout); i++ { tx.Vout[i].ScriptPubKey.Addresses = tmp.Vout[i].ScriptPubKey.Addresses - tx.Vout[i].Address = tmp.Vout[i].Address } } } diff --git a/bchain/types.go b/bchain/types.go index ba631310..50d69f9a 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -43,18 +43,11 @@ type ScriptPubKey struct { Addresses []string `json:"addresses"` } -type Address interface { - String() string - AreEqual(addr string) bool - InSlice(addrs []string) bool -} - type Vout struct { ValueSat big.Int JsonValue json.Number `json:"value"` N uint32 `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` - Address Address } // Tx is blockchain transaction @@ -164,12 +157,11 @@ type BlockChainParser interface { // AmountToBigInt converts amount in json.Number (string) to big.Int // it uses string operations to avoid problems with rounding AmountToBigInt(n json.Number) (big.Int, error) - // address id conversions - GetAddrIDFromVout(output *Vout) ([]byte, error) - GetAddrIDFromAddress(address string) ([]byte, error) - // address to output script conversions - AddressToOutputScript(address string) ([]byte, error) - OutputScriptToAddresses(script []byte) ([]string, error) + // address descriptor conversions + GetAddrDescFromVout(output *Vout) ([]byte, error) + GetAddrDescFromAddress(address string) ([]byte, error) + GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) + GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) // transactions PackedTxidLen() int PackTxid(txid string) ([]byte, error) diff --git a/db/rocksdb.go b/db/rocksdb.go index de498388..82c7d46d 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -24,7 +24,7 @@ import ( const refreshIterator = 5000000 const packedHeightBytes = 4 const dbVersion = 3 -const maxAddrIDLen = 1024 +const maxAddrDescLen = 1024 // RepairRocksDB calls RocksDb db repair function func RepairRocksDB(name string) error { @@ -161,13 +161,13 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f if glog.V(1) { glog.Infof("rocksdb: address get %s %d-%d ", address, lower, higher) } - addrID, err := d.chainParser.GetAddrIDFromAddress(address) + addrDesc, err := d.chainParser.GetAddrDescFromAddress(address) if err != nil { return err } - kstart := packAddressKey(addrID, lower) - kstop := packAddressKey(addrID, higher) + kstart := packAddressKey(addrDesc, lower) + kstop := packAddressKey(addrDesc, higher) it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) defer it.Close() @@ -281,24 +281,22 @@ type outpoint struct { } type TxInput struct { - addrID []byte + addrDesc []byte ValueSat big.Int } -func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, error) { - // TODO - we will need AddressesFromAddrID parser method, this will not work for ZCash - return p.OutputScriptToAddresses(ti.addrID) +func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { + return p.GetAddressesFromAddrDesc(ti.addrDesc) } type TxOutput struct { - addrID []byte + addrDesc []byte Spent bool ValueSat big.Int } -func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, error) { - // TODO - we will need AddressesFromAddrID parser method, this will not work for ZCash - return p.OutputScriptToAddresses(to.addrID) +func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { + return p.GetAddressesFromAddrDesc(to.addrDesc) } type TxAddresses struct { @@ -324,9 +322,9 @@ type blockTxs struct { inputs []outpoint } -func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText string) { - ad, err := d.chainParser.OutputScriptToAddresses(addrID) - had := hex.EncodeToString(addrID) +func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc []byte, logText string) { + ad, _, err := d.chainParser.GetAddressesFromAddrDesc(addrDesc) + had := hex.EncodeToString(addrDesc) if err != nil { glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", had, logText, valueSat.String(), err) } else { @@ -353,40 +351,40 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string for i, output := range tx.Vout { tao := &ta.Outputs[i] tao.ValueSat = output.ValueSat - addrID, err := d.chainParser.GetAddrIDFromVout(&output) - if err != nil || len(addrID) == 0 || len(addrID) > maxAddrIDLen { + addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) + if err != nil || len(addrDesc) == 0 || len(addrDesc) > maxAddrDescLen { if err != nil { // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) } } else { - glog.Infof("rocksdb: height %d, tx %v, vout %v, skipping addrID of length %d", block.Height, tx.Txid, i, len(addrID)) + glog.Infof("rocksdb: height %d, tx %v, vout %v, skipping addrDesc of length %d", block.Height, tx.Txid, i, len(addrDesc)) } continue } - tao.addrID = addrID - strAddrID := string(addrID) + tao.addrDesc = addrDesc + strAddrDesc := string(addrDesc) // check that the address was used already in this block - o, processed := addresses[strAddrID] + o, processed := addresses[strAddrDesc] if processed { // check that the address was already used in this tx processed = processedInTx(o, btxID) } - addresses[strAddrID] = append(o, outpoint{ + addresses[strAddrDesc] = append(o, outpoint{ btxID: btxID, index: int32(i), }) - ab, e := balances[strAddrID] + ab, e := balances[strAddrDesc] if !e { - ab, err = d.getAddrIDBalance(addrID) + ab, err = d.getAddrDescBalance(addrDesc) if err != nil { return err } if ab == nil { ab = &AddrBalance{} } - balances[strAddrID] = ab + balances[strAddrDesc] = ab } // add number of trx in balance only once, address can be multiple times in tx if !processed { @@ -433,38 +431,38 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string if ot.Spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } - tai.addrID = ot.addrID + tai.addrDesc = ot.addrDesc tai.ValueSat = ot.ValueSat // mark the output as spent in tx ot.Spent = true - if len(ot.addrID) == 0 { + if len(ot.addrDesc) == 0 { if !logged { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) logged = true } continue } - strAddrID := string(ot.addrID) + strAddrDesc := string(ot.addrDesc) // check that the address was used already in this block - o, processed := addresses[strAddrID] + o, processed := addresses[strAddrDesc] if processed { // check that the address was already used in this tx processed = processedInTx(o, spendingTxid) } - addresses[strAddrID] = append(o, outpoint{ + addresses[strAddrDesc] = append(o, outpoint{ btxID: spendingTxid, index: ^int32(i), }) - ab, e := balances[strAddrID] + ab, e := balances[strAddrDesc] if !e { - ab, err = d.getAddrIDBalance(ot.addrID) + ab, err = d.getAddrDescBalance(ot.addrDesc) if err != nil { return err } if ab == nil { ab = &AddrBalance{} } - balances[strAddrID] = ab + balances[strAddrDesc] = ab } // add number of trx in balance only once, address can be multiple times in tx if !processed { @@ -472,7 +470,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string } ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat) if ab.BalanceSat.Sign() < 0 { - d.resetValueSatToZero(&ab.BalanceSat, ot.addrID, "balance") + d.resetValueSatToZero(&ab.BalanceSat, ot.addrDesc, "balance") } ab.SentSat.Add(&ab.SentSat, &ot.ValueSat) } @@ -490,8 +488,8 @@ func processedInTx(o []outpoint, btxID []byte) bool { } func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error { - for addrID, outpoints := range addresses { - ba := []byte(addrID) + for addrDesc, outpoints := range addresses { + ba := []byte(addrDesc) key := packAddressKey(ba, height) val := d.packOutpoints(outpoints) wb.PutCF(d.cfh[cfAddresses], key, val) @@ -512,17 +510,17 @@ func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*TxAd func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBalance) error { // allocate buffer big enough for number of txs + 2 bigints buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) - for addrID, ab := range abm { + for addrDesc, ab := range abm { // balance with 0 transactions is removed from db - happens in disconnect if ab == nil || ab.Txs <= 0 { - wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrID)) + wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrDesc)) } else { l := packVaruint(uint(ab.Txs), buf) ll := packBigint(&ab.SentSat, buf[l:]) l += ll ll = packBigint(&ab.BalanceSat, buf[l:]) l += ll - wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + wb.PutCF(d.cfh[cfAddressBalance], []byte(addrDesc), buf[:l]) } } return nil @@ -611,8 +609,8 @@ func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { return bt, nil } -func (d *RocksDB) getAddrIDBalance(addrID []byte) (*AddrBalance, error) { - val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) +func (d *RocksDB) getAddrDescBalance(addrDesc []byte) (*AddrBalance, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrDesc) if err != nil { return nil, err } @@ -634,11 +632,11 @@ func (d *RocksDB) getAddrIDBalance(addrID []byte) (*AddrBalance, error) { // GetAddressBalance returns address balance for an address or nil if address not found func (d *RocksDB) GetAddressBalance(address string) (*AddrBalance, error) { - addrID, err := d.chainParser.GetAddrIDFromAddress(address) + addrDesc, err := d.chainParser.GetAddrDescFromAddress(address) if err != nil { return nil, err } - return d.getAddrIDBalance(addrID) + return d.getAddrDescBalance(addrDesc) } func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { @@ -682,23 +680,23 @@ func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { } func appendTxInput(txi *TxInput, buf []byte, varBuf []byte) []byte { - la := len(txi.addrID) + la := len(txi.addrDesc) l := packVaruint(uint(la), varBuf) buf = append(buf, varBuf[:l]...) - buf = append(buf, txi.addrID...) + buf = append(buf, txi.addrDesc...) l = packBigint(&txi.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf } func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { - la := len(txo.addrID) + la := len(txo.addrDesc) if txo.Spent { la = ^la } l := packVarint(la, varBuf) buf = append(buf, varBuf[:l]...) - buf = append(buf, txo.addrID...) + buf = append(buf, txo.addrDesc...) l = packBigint(&txo.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf @@ -725,8 +723,8 @@ func unpackTxAddresses(buf []byte) (*TxAddresses, error) { func unpackTxInput(ti *TxInput, buf []byte) int { al, l := unpackVaruint(buf) - ti.addrID = make([]byte, al) - copy(ti.addrID, buf[l:l+int(al)]) + ti.addrDesc = make([]byte, al) + copy(ti.addrDesc, buf[l:l+int(al)]) al += uint(l) ti.ValueSat, l = unpackBigint(buf[al:]) return l + int(al) @@ -738,8 +736,8 @@ func unpackTxOutput(to *TxOutput, buf []byte) int { to.Spent = true al = ^al } - to.addrID = make([]byte, al) - copy(to.addrID, buf[l:l+al]) + to.addrDesc = make([]byte, al) + copy(to.addrDesc, buf[l:l+al]) al += l to.ValueSat, l = unpackBigint(buf[al:]) return l + al @@ -792,13 +790,13 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { return outpoints, p, nil } -func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { - if len(addrID) > 0 { - if len(addrID) > maxAddrIDLen { - glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) +func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc []byte, btxid []byte, vout int32, bh uint32) error { + if len(addrDesc) > 0 { + if len(addrDesc) > maxAddrDescLen { + glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc)) } else { - strAddrID := string(addrID) - records[strAddrID] = append(records[strAddrID], outpoint{ + strAddrDesc := string(addrDesc) + records[strAddrDesc] = append(records[strAddrDesc], outpoint{ btxID: btxid, index: vout, }) @@ -819,15 +817,15 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. return err } for _, output := range tx.Vout { - addrID, err := d.chainParser.GetAddrIDFromVout(&output) + addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) if err != nil { // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) } continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(output.N), block.Height) + err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(output.N), block.Height) if err != nil { return err } @@ -835,20 +833,20 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // store inputs in format txid ^index for _, input := range tx.Vin { for i, a := range input.Addresses { - addrID, err := d.chainParser.GetAddrIDFromAddress(a) + addrDesc, err := d.chainParser.GetAddrDescFromAddress(a) if err != nil { - glog.Warningf("rocksdb: addrID: %v - %d %s", err, block.Height, addrID) + glog.Warningf("rocksdb: addrDesc: %v - %d %s", err, block.Height, addrDesc) continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(^i), block.Height) + err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(^i), block.Height) if err != nil { return err } } } } - for addrID, outpoints := range addresses { - key := packAddressKey([]byte(addrID), block.Height) + for addrDesc, outpoints := range addresses { + key := packAddressKey([]byte(addrDesc), block.Height) switch op { case opInsert: val := d.packOutpoints(outpoints) @@ -1024,12 +1022,12 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *TxAddresses, txAddressesToUpdate map[string]*TxAddresses, balances map[string]*AddrBalance) error { addresses := make(map[string]struct{}) - getAddressBalance := func(addrID []byte) (*AddrBalance, error) { + getAddressBalance := func(addrDesc []byte) (*AddrBalance, error) { var err error - s := string(addrID) + s := string(addrDesc) b, fb := balances[s] if !fb { - b, err = d.getAddrIDBalance(addrID) + b, err = d.getAddrDescBalance(addrDesc) if err != nil { return nil, err } @@ -1038,13 +1036,13 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, return b, nil } for i, t := range txa.Inputs { - if len(t.addrID) > 0 { - s := string(t.addrID) + if len(t.addrDesc) > 0 { + s := string(t.addrDesc) _, exist := addresses[s] if !exist { addresses[s] = struct{}{} } - b, err := getAddressBalance(t.addrID) + b, err := getAddressBalance(t.addrDesc) if err != nil { return err } @@ -1055,12 +1053,12 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } b.SentSat.Sub(&b.SentSat, &t.ValueSat) if b.SentSat.Sign() < 0 { - d.resetValueSatToZero(&b.SentSat, t.addrID, "sent amount") + d.resetValueSatToZero(&b.SentSat, t.addrDesc, "sent amount") } b.BalanceSat.Add(&b.BalanceSat, &t.ValueSat) } else { - ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) - had := hex.EncodeToString(t.addrID) + ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.addrDesc) + had := hex.EncodeToString(t.addrDesc) glog.Warningf("Balance for address %s (%s) not found", ad, had) } s = string(inputs[i].btxID) @@ -1076,13 +1074,13 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } } for _, t := range txa.Outputs { - if len(t.addrID) > 0 { - s := string(t.addrID) + if len(t.addrDesc) > 0 { + s := string(t.addrDesc) _, exist := addresses[s] if !exist { addresses[s] = struct{}{} } - b, err := getAddressBalance(t.addrID) + b, err := getAddressBalance(t.addrDesc) if err != nil { return err } @@ -1093,11 +1091,11 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } b.BalanceSat.Sub(&b.BalanceSat, &t.ValueSat) if b.BalanceSat.Sign() < 0 { - d.resetValueSatToZero(&b.BalanceSat, t.addrID, "balance") + d.resetValueSatToZero(&b.BalanceSat, t.addrDesc, "balance") } } else { - ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) - had := hex.EncodeToString(t.addrID) + ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.addrDesc) + had := hex.EncodeToString(t.addrDesc) glog.Warningf("Balance for address %s (%s) not found", ad, had) } } @@ -1429,10 +1427,10 @@ func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) er // Helpers -func packAddressKey(addrID []byte, height uint32) []byte { +func packAddressKey(addrDesc []byte, height uint32) []byte { bheight := packUint(height) - buf := make([]byte, 0, len(addrID)+len(bheight)) - buf = append(buf, addrID...) + buf := make([]byte, 0, len(addrDesc)+len(bheight)) + buf = append(buf, addrDesc...) buf = append(buf, bheight...) return buf } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 3b21b93d..e1785fda 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -1,4 +1,4 @@ -// build unittest +// +build unittest package db @@ -59,7 +59,7 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { if addr == "" { return "" } - b, err := d.chainParser.AddressToOutputScript(addr) + b, err := d.chainParser.GetAddrDescFromAddress(addr) if err != nil { t.Fatal(err) } @@ -816,22 +816,22 @@ func TestRocksDB_Index_UTXO(t *testing.T) { Height: 225494, Inputs: []TxInput{ { - addrID: addressToOutput(addr3, d.chainParser), + addrDesc: addressToAddrDesc(addr3, d.chainParser), ValueSat: *satB1T2A3, }, { - addrID: addressToOutput(addr2, d.chainParser), + addrDesc: addressToAddrDesc(addr2, d.chainParser), ValueSat: *satB1T1A2, }, }, Outputs: []TxOutput{ { - addrID: addressToOutput(addr6, d.chainParser), + addrDesc: addressToAddrDesc(addr6, d.chainParser), Spent: true, ValueSat: *satB2T1A6, }, { - addrID: addressToOutput(addr7, d.chainParser), + addrDesc: addressToAddrDesc(addr7, d.chainParser), Spent: false, ValueSat: *satB2T1A7, }, @@ -840,7 +840,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { if !reflect.DeepEqual(ta, taw) { t.Errorf("GetTxAddresses() = %+v, want %+v", ta, taw) } - ia, err := ta.Inputs[0].Addresses(d.chainParser) + ia, _, err := ta.Inputs[0].Addresses(d.chainParser) if err != nil { t.Fatal(err) } @@ -979,8 +979,8 @@ func Test_packBigint_unpackBigint(t *testing.T) { } } -func addressToOutput(addr string, parser bchain.BlockChainParser) []byte { - b, err := parser.AddressToOutputScript(addr) +func addressToAddrDesc(addr string, parser bchain.BlockChainParser) []byte { + b, err := parser.GetAddrDescFromAddress(addr) if err != nil { panic(err) } @@ -1001,17 +1001,17 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 123, Inputs: []TxInput{ { - addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + addrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), ValueSat: *big.NewInt(0), }, { - addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), + addrDesc: addressToAddrDesc("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), ValueSat: *big.NewInt(1234123421342341234), }, }, Outputs: []TxOutput{ { - addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + addrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), ValueSat: *big.NewInt(1), Spent: true, }, @@ -1025,39 +1025,39 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 12345, Inputs: []TxInput{ { - addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + addrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), ValueSat: *big.NewInt(9011000000), }, { - addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + addrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), ValueSat: *big.NewInt(8011000000), }, { - addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + addrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), ValueSat: *big.NewInt(7011000000), }, }, Outputs: []TxOutput{ { - addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + addrDesc: addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), ValueSat: *big.NewInt(5011000000), Spent: true, }, { - addrID: addressToOutput("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + addrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), ValueSat: *big.NewInt(6011000000), }, { - addrID: addressToOutput("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + addrDesc: addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), ValueSat: *big.NewInt(7011000000), Spent: true, }, { - addrID: addressToOutput("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + addrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), ValueSat: *big.NewInt(999900000), }, { - addrID: addressToOutput("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + addrDesc: addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), ValueSat: *big.NewInt(5000000000), Spent: true, }, @@ -1071,17 +1071,17 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 123456789, Inputs: []TxInput{ { - addrID: []byte{}, + addrDesc: []byte{}, ValueSat: *big.NewInt(1234), }, }, Outputs: []TxOutput{ { - addrID: []byte{}, + addrDesc: []byte{}, ValueSat: *big.NewInt(5678), }, { - addrID: []byte{}, + addrDesc: []byte{}, ValueSat: *big.NewInt(98), Spent: true, }, diff --git a/server/public.go b/server/public.go index 36556850..9b9800b7 100644 --- a/server/public.go +++ b/server/public.go @@ -338,8 +338,7 @@ func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, err if ec != nil { page = 0 } - addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page, txsOnPage, false) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsOnPage, false) if err != nil { return errorTpl, nil, err } @@ -439,8 +438,7 @@ func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { if ec != nil { page = 0 } - addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page, txsInAPI, true) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsInAPI, true) } return address, err } diff --git a/server/socketio.go b/server/socketio.go index 74fbd67b..9d36f2e4 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -306,6 +306,29 @@ func txToResTx(tx *bchain.Tx, height int, hi []txInputs, ho []txOutputs) resTx { } } +func addressInSlice(s, t []string) string { + for _, sa := range s { + for _, ta := range t { + if ta == sa { + return sa + } + } + } + return "" +} + +func (s *SocketIoServer) getAddressesFromVout(vout *bchain.Vout) ([]string, error) { + addrDesc, err := s.chainParser.GetAddrDescFromVout(vout) + if err != nil { + return nil, err + } + voutAddr, _, err := s.chainParser.GetAddressesFromAddrDesc(addrDesc) + if err != nil { + return nil, err + } + return voutAddr, nil +} + func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res resultGetAddressHistory, err error) { txr, err := s.getAddressTxids(addr, opts) if err != nil { @@ -333,19 +356,23 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r Satoshis: vout.ValueSat.String(), Script: &aoh, } - if vout.Address != nil { - a := vout.Address.String() - ao.Address = &a - if vout.Address.InSlice(addr) { - hi, ok := ads[a] - if ok { - hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) - } else { - hi := addressHistoryIndexes{} - hi.InputIndexes = make([]int, 0) - hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) - ads[a] = hi - } + voutAddr, err := s.getAddressesFromVout(&vout) + if err != nil { + return res, err + } + if len(voutAddr) > 0 { + ao.Address = &voutAddr[0] + } + a := addressInSlice(voutAddr, addr) + if a != "" { + hi, ok := ads[a] + if ok { + hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) + } else { + hi := addressHistoryIndexes{} + hi.InputIndexes = make([]int, 0) + hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) + ads[a] = hi } } ho = append(ho, ao) @@ -587,9 +614,12 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai } if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] - if vout.Address != nil { - a := vout.Address.String() - ai.Address = &a + voutAddr, err := s.getAddressesFromVout(vout) + if err != nil { + return res, err + } + if len(voutAddr) > 0 { + ai.Address = &voutAddr[0] } ai.Satoshis = vout.ValueSat.String() } @@ -602,9 +632,12 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai Satoshis: vout.ValueSat.String(), Script: &aos, } - if vout.Address != nil { - a := vout.Address.String() - ao.Address = &a + voutAddr, err := s.getAddressesFromVout(&vout) + if err != nil { + return res, err + } + if len(voutAddr) > 0 { + ao.Address = &voutAddr[0] } ho = append(ho, ao) } diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 87ebd673..2c70c912 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -16,13 +16,16 @@ {{range $vin := $tx.Vin}} - {{if $vin.Txid}} - - {{if $vin.Addr}}{{if eq $vin.Addr $addr}}{{$vin.Addr}}{{else}} - {{$vin.Addr}}{{end}}{{else}}Unparsed address{{end}} + {{range $a := $vin.Addresses}} + + {{if eq $a $addr}}{{$a}}{{else}} + {{$a}}{{end}} - {{formatAmount $vin.Value}} {{$cs}} - {{else}}No Inputs (Newly Generated Coins){{end}} + {{else}} + No Inputs (Newly Generated Coins) + {{end}} {{if $vin.Addresses}} + {{formatAmount $vin.Value}} {{$cs}} + {{end}} {{end}} From b9c0ca37b18c8a05acf16295a40233deb495cab5 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 29 Aug 2018 19:48:44 +0200 Subject: [PATCH 037/123] Fix monacoin tests --- ...onacoin_test.go => monacoinparser_test.go} | 0 bchain/tests/rpc/testdata/Monacoin.json | 54 +++++++++++++++++++ 2 files changed, 54 insertions(+) rename bchain/coins/monacoin/{monacoin_test.go => monacoinparser_test.go} (100%) create mode 100644 bchain/tests/rpc/testdata/Monacoin.json diff --git a/bchain/coins/monacoin/monacoin_test.go b/bchain/coins/monacoin/monacoinparser_test.go similarity index 100% rename from bchain/coins/monacoin/monacoin_test.go rename to bchain/coins/monacoin/monacoinparser_test.go diff --git a/bchain/tests/rpc/testdata/Monacoin.json b/bchain/tests/rpc/testdata/Monacoin.json new file mode 100644 index 00000000..96a03299 --- /dev/null +++ b/bchain/tests/rpc/testdata/Monacoin.json @@ -0,0 +1,54 @@ +{ + "blockHeight": 1205643, + "blockHash": "60c80ce4f0a90f7b217bef8fab32ce2e499a4dc2067f4a6062048e3b940d76c6", + "blockTxs": [ + "38b4a919ab0f8b4570e1f26717740846b7994ba22f94c51d96eb32bc45c5886e", + "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298", + "c2bdcc759c1d4e6bb3eee859cc7400bc376479e30bc8328f31864aad92d52ade", + "8a26deeda15f1ddc3234acafb0da9829bf334ea108e359f11d8f281ae42512f8" + ], + "txDetails": { + "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298": { + "hex": "0200000002d9d3592e550618b41347bd992d33b9eaebe7e26eb21df49bda5372ce1711f8e5010000006b483045022100b4fb223759753d82fd5ce833e84f62904c086b51fd2be11d7e3e9f183e2442850220079f8b3d6555610ff98da3216fa117bcb49f98e610a38471d69566ce7bc5770801210362c5e60c40f0b588216310dbb2f4a46e86038ba597f33d023ad64e66ff2e972bfeffffff4f4e9b373e6cd134d2f5f45bf3645a00c2689472ad66aa37ae022eef3dd6b16a010000006a47304402203dd8510301d68bc21169f953d2996d602f8411a309c494392b6f2bf85e5d58f102206fa2db245d15021f3b556d10747cfd8eba100fd19dfa1a0a9c5cebc58d7c22f6012103c65394a4c6705e3a13f6305e7997fb68feb192acce312a8ba144994b9a33507bfeffffff02bbe52c00000000001976a914eb0f5829a547f30b3d488ab38b425c1d93afb4b888ac00f90295000000001976a914883f0723e1c93db538d02c7a8d318322e655b59888ac89651200", + "txid": "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298", + "blocktime": 1514835214, + "time": 1514835214, + "locktime": 1205641, + "version": 2, + "vin": [ + { + "txid": "e5f81117ce7253da9bf41db26ee2e7ebeab9332d99bd4713b41806552e59d3d9", + "vout": 1, + "scriptSig": { + "hex": "483045022100b4fb223759753d82fd5ce833e84f62904c086b51fd2be11d7e3e9f183e2442850220079f8b3d6555610ff98da3216fa117bcb49f98e610a38471d69566ce7bc5770801210362c5e60c40f0b588216310dbb2f4a46e86038ba597f33d023ad64e66ff2e972b" + }, + "sequence": 4294967294 + }, + { + "txid": "6ab1d63def2e02ae37aa66ad729468c2005a64f35bf4f5d234d16c3e379b4e4f", + "vout": 1, + "scriptSig": { + "hex": "47304402203dd8510301d68bc21169f953d2996d602f8411a309c494392b6f2bf85e5d58f102206fa2db245d15021f3b556d10747cfd8eba100fd19dfa1a0a9c5cebc58d7c22f6012103c65394a4c6705e3a13f6305e7997fb68feb192acce312a8ba144994b9a33507b" + }, + "sequence": 4294967294 + } + ], + "vout": [ + { + "value": 0.02942395, + "n": 0, + "scriptPubKey": { + "hex": "76a914eb0f5829a547f30b3d488ab38b425c1d93afb4b888ac" + } + }, + { + "value": 25.00000000, + "n": 1, + "scriptPubKey": { + "hex": "76a914883f0723e1c93db538d02c7a8d318322e655b59888ac" + } + } + ] + } + } +} \ No newline at end of file From bd88ab4094f8971fa5504ce8650624b8e5bfdb5b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 29 Aug 2018 19:58:38 +0200 Subject: [PATCH 038/123] Add monacoin to rpc test config.json --- bchain/tests/rpc/config.json | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/bchain/tests/rpc/config.json b/bchain/tests/rpc/config.json index e4019e07..8bbe587e 100644 --- a/bchain/tests/rpc/config.json +++ b/bchain/tests/rpc/config.json @@ -1,11 +1,16 @@ { + "Bitcoin": { + "url": "http://localhost:8030", + "user": "rpc", + "pass": "rpc" + }, "Bcash": { "url": "http://localhost:8031", "user": "rpc", "pass": "rpc" }, - "Bitcoin": { - "url": "http://localhost:8030", + "Zcash": { + "url": "http://localhost:8032", "user": "rpc", "pass": "rpc" }, @@ -19,8 +24,8 @@ "user": null, "pass": null }, - "Zcash": { - "url": "http://localhost:8032", + "Namecoin": { + "url": "http://localhost:8039", "user": "rpc", "pass": "rpc" }, @@ -29,8 +34,13 @@ "user": "rpc", "pass": "rpc" }, - "Namecoin": { - "url": "http://localhost:8039", + "Monacoin": { + "url": "http://localhost:8041", + "user": "rpc", + "pass": "rpc" + }, + "Bitcoin Testnet": { + "url": "http://localhost:18030", "user": "rpc", "pass": "rpc" }, @@ -39,8 +49,8 @@ "user": "rpc", "pass": "rpc" }, - "Bitcoin Testnet": { - "url": "http://localhost:18030", + "Zcash Testnet": { + "url": "http://localhost:18032", "user": "rpc", "pass": "rpc" }, @@ -53,10 +63,5 @@ "url": "ws://localhost:18036", "user": null, "pass": null - }, - "Zcash Testnet": { - "url": "http://localhost:18032", - "user": "rpc", - "pass": "rpc" } -} +} \ No newline at end of file From 06ab4fb9f8172d12db72a5c68faa8f9544e55f4b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 29 Aug 2018 21:16:23 +0200 Subject: [PATCH 039/123] Fix rpc tests --- bchain/tests/rpc/data.go | 8 ++++---- bchain/tests/rpc/rpc.go | 4 +++- bchain/tests/rpc/testdata/Bcash.json | 1 + bchain/tests/rpc/testdata/Bcash_Testnet.json | 1 + bchain/tests/rpc/testdata/Bitcoin.json | 1 + bchain/tests/rpc/testdata/Bitcoin_Testnet.json | 1 + bchain/tests/rpc/testdata/Dash.json | 1 + bchain/tests/rpc/testdata/Dash_Testnet.json | 3 ++- bchain/tests/rpc/testdata/Ethereum_Testnet.json | 1 + bchain/tests/rpc/testdata/Monacoin.json | 1 + bchain/tests/rpc/testdata/Namecoin.json | 1 + bchain/tests/rpc/testdata/Vertcoin.json | 1 + bchain/tests/rpc/testdata/Zcash.json | 1 + bchain/tests/rpc/testdata/Zcash_Testnet.json | 1 + 14 files changed, 20 insertions(+), 6 deletions(-) diff --git a/bchain/tests/rpc/data.go b/bchain/tests/rpc/data.go index 68d69acf..d855ed48 100644 --- a/bchain/tests/rpc/data.go +++ b/bchain/tests/rpc/data.go @@ -67,10 +67,10 @@ func LoadTestConfig(coin string) (*TestConfig, error) { func LoadRPCConfig(coin string) (json.RawMessage, error) { t := `{ "coin_name": "%s", - "rpcURL": "%s", - "rpcUser": "%s", - "rpcPass": "%s", - "rpcTimeout": 25, + "rpc_url": "%s", + "rpc_user": "%s", + "rpc_pass": "%s", + "rpc_timeout": 25, "parse": true }` diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go index bc7960a8..675e6015 100644 --- a/bchain/tests/rpc/rpc.go +++ b/bchain/tests/rpc/rpc.go @@ -21,6 +21,7 @@ type TestConfig struct { type TestData struct { BlockHeight uint32 `json:"blockHeight"` BlockHash string `json:"blockHash"` + BlockTime int64 `json:"blockTime"` BlockTxs []string `json:"blockTxs"` TxDetails map[string]*bchain.Tx `json:"txDetails"` } @@ -432,6 +433,7 @@ func (rt *Test) TestGetBlockHeader(t *testing.T) { want := &bchain.BlockHeader{ Hash: rt.TestData.BlockHash, Height: rt.TestData.BlockHeight, + Time: rt.TestData.BlockTime, } got, err := rt.Client.GetBlockHeader(rt.TestData.BlockHash) @@ -448,6 +450,6 @@ func (rt *Test) TestGetBlockHeader(t *testing.T) { got.Prev, got.Next = "", "" if !reflect.DeepEqual(got, want) { - t.Errorf("GetBlockHeader() got=%v, want=%v", got, want) + t.Errorf("GetBlockHeader() got=%+v, want=%+v", got, want) } } diff --git a/bchain/tests/rpc/testdata/Bcash.json b/bchain/tests/rpc/testdata/Bcash.json index 9b1f647b..5459546e 100644 --- a/bchain/tests/rpc/testdata/Bcash.json +++ b/bchain/tests/rpc/testdata/Bcash.json @@ -1,6 +1,7 @@ { "blockHeight": 538226, "blockHash": "0000000000000000008f48dad3a9efb25275fe36bb31b460e775af57783cc9af", + "blockTime": 1531135480, "blockTxs": [ "77360353ff628d53534201712db7a7689a3f9318cad240fbc093dfb4778fdf84", "82d95972a276b4197806ec0d9e3e56b66523849b2a7597ac509f4dfc8f7db4fd", diff --git a/bchain/tests/rpc/testdata/Bcash_Testnet.json b/bchain/tests/rpc/testdata/Bcash_Testnet.json index c33176d3..96b9f476 100644 --- a/bchain/tests/rpc/testdata/Bcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Bcash_Testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 1241851, "blockHash": "00000000ff54973b45ac277fc9e92408479703d785eae0e96508bc5aa792d621", + "blockTime": 1529571678, "blockTxs": [ "0cbcd28da300a850fd922caf633f2aaca0fd0a2d4ce46c6f07314c644287527a", "32fe2a3a60eb952d6ade53b7030ddb6ad8af5797391857baaad37c5b20f3b5ca" diff --git a/bchain/tests/rpc/testdata/Bitcoin.json b/bchain/tests/rpc/testdata/Bitcoin.json index 5cf34ec8..ae426860 100644 --- a/bchain/tests/rpc/testdata/Bitcoin.json +++ b/bchain/tests/rpc/testdata/Bitcoin.json @@ -1,6 +1,7 @@ { "blockHeight": 529150, "blockHash": "00000000000000000035835503f43c878ebb643f3b40bdfd0dfda760da74e73c", + "blockTime": 1529915213, "blockTxs": [ "8dd1379174e262d12a32d217e87a7caf09fa1b9e48a6fe010cac219f18c6de58", "5fce44793b328ca5f142caadbf29efc78a0059d7a6379dff81fc6447b519a7c3", diff --git a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json index cab2c840..ed0cde17 100644 --- a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json +++ b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 1325168, "blockHash": "000000000000004ed0834f3de922e66d024ec4da9fcc2da17be61369cb6dc041", + "blockTime": 1528788394, "blockTxs": [ "e1179f205aabbf48dc2ce4ebd9ed255571b0578e4de551f6574a50cb81120007", "00a5aa2891d41af9eb1dc30c940f142a609ecab8f370eb0874ba7d32252d1b1b", diff --git a/bchain/tests/rpc/testdata/Dash.json b/bchain/tests/rpc/testdata/Dash.json index f5ab09b6..d43f66db 100644 --- a/bchain/tests/rpc/testdata/Dash.json +++ b/bchain/tests/rpc/testdata/Dash.json @@ -1,6 +1,7 @@ { "blockHeight": 894503, "blockHash": "0000000000000026e2b7c7bb478f0ee846dd976ac6b97c3f3f7c8c65beab88a4", + "blockTime": 1530189699, "blockTxs": [ "6ea3a7fee4c40170d8782719d6fbeafd4cabb830f880d91bcf34811ad79e57bb", "c269940d8e46d94ff61a4adec8aa6dfb13803723a95a5b61c7831d3aea308cbc", diff --git a/bchain/tests/rpc/testdata/Dash_Testnet.json b/bchain/tests/rpc/testdata/Dash_Testnet.json index 64509772..c8a6299c 100644 --- a/bchain/tests/rpc/testdata/Dash_Testnet.json +++ b/bchain/tests/rpc/testdata/Dash_Testnet.json @@ -1,7 +1,8 @@ { "blockHeight": 139521, "blockHash": "000000000296ea05e13b5479f6c041de575eec90759f58f57d311a8918b0af17", - "blockTxs": [ + "blockTime": 1528713762, + "blockTxs": [ "cfe2d2c5bd9929349a9d8f3d8f2423f4f0c9b408ed41b3002be5e7437a20aa7d", "3f7ac6c55c0b383feb17715aef3cebf29182039fc42577d7c517b2fc04096942", "3f296b929f6ee4f0ad3d6fce35873d6aeacbdf06aaf2abe384beb6e3f2a4dd03", diff --git a/bchain/tests/rpc/testdata/Ethereum_Testnet.json b/bchain/tests/rpc/testdata/Ethereum_Testnet.json index b7cd76cb..a550e3f9 100644 --- a/bchain/tests/rpc/testdata/Ethereum_Testnet.json +++ b/bchain/tests/rpc/testdata/Ethereum_Testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 2870000, "blockHash": "0xeccd6b0031015a19cb7d4e10f28590ba65a6a54ad1baa322b50fe5ad16903895", + "blockTime": 1521515026, "blockTxs": [ "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", diff --git a/bchain/tests/rpc/testdata/Monacoin.json b/bchain/tests/rpc/testdata/Monacoin.json index 96a03299..2e78ebe6 100644 --- a/bchain/tests/rpc/testdata/Monacoin.json +++ b/bchain/tests/rpc/testdata/Monacoin.json @@ -1,6 +1,7 @@ { "blockHeight": 1205643, "blockHash": "60c80ce4f0a90f7b217bef8fab32ce2e499a4dc2067f4a6062048e3b940d76c6", + "blockTime": 1514835214, "blockTxs": [ "38b4a919ab0f8b4570e1f26717740846b7994ba22f94c51d96eb32bc45c5886e", "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298", diff --git a/bchain/tests/rpc/testdata/Namecoin.json b/bchain/tests/rpc/testdata/Namecoin.json index 5a56c972..489f0c53 100644 --- a/bchain/tests/rpc/testdata/Namecoin.json +++ b/bchain/tests/rpc/testdata/Namecoin.json @@ -1,6 +1,7 @@ { "blockHeight": 404680, "blockHash": "920fe53b840111f7e593d93ba58dc54e043e10f8fa4a678e86a98f5cb5b29614", + "blockTime": 1530003649, "blockTxs": [ "80b8477d10df9ece7d8dde2d30817e2855af1fb66b7a9ac860e592118ae33f5f", "afcd8e3638b11b1ce52055474dcce78c4129e95ebee3305574acda48deea8a65", diff --git a/bchain/tests/rpc/testdata/Vertcoin.json b/bchain/tests/rpc/testdata/Vertcoin.json index d5c5383e..fb18cacb 100644 --- a/bchain/tests/rpc/testdata/Vertcoin.json +++ b/bchain/tests/rpc/testdata/Vertcoin.json @@ -1,6 +1,7 @@ { "blockHeight": 952235, "blockHash": "b2787dd022e3aa65b63dbf08af2c9bb4d4a362d95e3328c02743a5c8d75acb36", + "blockTime": 1529932850, "blockTxs": [ "366eca05fa8579465d8822ad6462762120b26239201a34981e5f9d9efac3cc31", "e74c247a5a77d4edd96a5dbb2930c74d9ab550affde991731f78f3e3a2f4b559", diff --git a/bchain/tests/rpc/testdata/Zcash.json b/bchain/tests/rpc/testdata/Zcash.json index 346d9f96..f8b83f80 100644 --- a/bchain/tests/rpc/testdata/Zcash.json +++ b/bchain/tests/rpc/testdata/Zcash.json @@ -1,6 +1,7 @@ { "blockHeight": 349410, "blockHash": "000000000101b45343aeda139d06d7f188393a5cf4adabfa191cb1828d67f6bd", + "blockTime": 1530264033, "blockTxs": [ "6ebf0114454ba776f878cf726be3cb65964b21fad8dfca5d2ee69bd4a32cfa97", "3de894528d5f8dbe14f726244be70a88f9b738020215c91b2893fa5a46f48c14", diff --git a/bchain/tests/rpc/testdata/Zcash_Testnet.json b/bchain/tests/rpc/testdata/Zcash_Testnet.json index 5d78989c..8ed594c1 100644 --- a/bchain/tests/rpc/testdata/Zcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Zcash_Testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 251102, "blockHash": "001335906f981bbf0633e124e2fa8afef3d882e34a0306a4e0c55162e57e673d", + "blockTime": 1528781777, "blockTxs": [ "f02aa1c4c86e1d0cef6ccbbc48b2b7b38355bc3612d8f77dd58d04be1ec6ba19", "a9f7cc34d7e272d2d9fb68cfa1c1941e338f377e6e426ae2fea1c12616d89c63", From d3608fb34c347072f8dafc3d660bbdf08e7f159e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 29 Aug 2018 21:29:09 +0200 Subject: [PATCH 040/123] Add dogecoin and litecoin rpc test --- bchain/coins/dogecoin/dogecoinrpc_test.go | 72 +++++++++++++++++++++++ bchain/coins/litecoin/litecoinrpc_test.go | 72 +++++++++++++++++++++++ bchain/tests/rpc/config.json | 10 ++++ bchain/tests/rpc/testdata/Dogecoin.json | 53 +++++++++++++++++ bchain/tests/rpc/testdata/Litecoin.json | 49 +++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 bchain/coins/dogecoin/dogecoinrpc_test.go create mode 100644 bchain/coins/litecoin/litecoinrpc_test.go create mode 100644 bchain/tests/rpc/testdata/Dogecoin.json create mode 100644 bchain/tests/rpc/testdata/Litecoin.json diff --git a/bchain/coins/dogecoin/dogecoinrpc_test.go b/bchain/coins/dogecoin/dogecoinrpc_test.go new file mode 100644 index 00000000..17ffb0ef --- /dev/null +++ b/bchain/coins/dogecoin/dogecoinrpc_test.go @@ -0,0 +1,72 @@ +// +build integration + +package dogecoin + +import ( + "blockbook/bchain" + "blockbook/bchain/tests/rpc" + "encoding/json" + "flag" + "os" + "testing" +) + +func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { + c, err := NewDogecoinRPC(cfg, nil) + if err != nil { + return nil, err + } + cli := c.(*DogecoinRPC) + cli.Parser = NewDogecoinParser(GetChainParams("main"), cli.ChainConfig) + if err != nil { + return nil, err + } + cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) + return cli, nil +} + +var rpcTest *rpc.Test + +func TestMain(m *testing.M) { + flag.Parse() + t, err := rpc.NewTest("Dogecoin", getRPCClient) + if err != nil { + panic(err) + } + + rpcTest = t + + os.Exit(m.Run()) +} + +func TestDogecoinRPC_GetBlockHash(t *testing.T) { + rpcTest.TestGetBlockHash(t) +} + +func TestDogecoinRPC_GetBlock(t *testing.T) { + rpcTest.TestGetBlock(t) +} + +func TestDogecoinRPC_GetTransaction(t *testing.T) { + rpcTest.TestGetTransaction(t) +} + +func TestDogecoinRPC_GetTransactionForMempool(t *testing.T) { + rpcTest.TestGetTransactionForMempool(t) +} + +func TestDogecoinRPC_MempoolSync(t *testing.T) { + rpcTest.TestMempoolSync(t) +} + +func TestDogecoinRPC_GetMempoolEntry(t *testing.T) { + rpcTest.TestGetMempoolEntry(t) +} + +func TestDogecoinRPC_EstimateSmartFee(t *testing.T) { + rpcTest.TestEstimateSmartFee(t) +} + +func TestDogecoinRPC_EstimateFee(t *testing.T) { + rpcTest.TestEstimateFee(t) +} diff --git a/bchain/coins/litecoin/litecoinrpc_test.go b/bchain/coins/litecoin/litecoinrpc_test.go new file mode 100644 index 00000000..33a67b91 --- /dev/null +++ b/bchain/coins/litecoin/litecoinrpc_test.go @@ -0,0 +1,72 @@ +// +build integration + +package litecoin + +import ( + "blockbook/bchain" + "blockbook/bchain/tests/rpc" + "encoding/json" + "flag" + "os" + "testing" +) + +func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { + c, err := NewLitecoinRPC(cfg, nil) + if err != nil { + return nil, err + } + cli := c.(*LitecoinRPC) + cli.Parser = NewLitecoinParser(GetChainParams("main"), cli.ChainConfig) + if err != nil { + return nil, err + } + cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) + return cli, nil +} + +var rpcTest *rpc.Test + +func TestMain(m *testing.M) { + flag.Parse() + t, err := rpc.NewTest("Litecoin", getRPCClient) + if err != nil { + panic(err) + } + + rpcTest = t + + os.Exit(m.Run()) +} + +func TestLitecoinRPC_GetBlockHash(t *testing.T) { + rpcTest.TestGetBlockHash(t) +} + +func TestLitecoinRPC_GetBlock(t *testing.T) { + rpcTest.TestGetBlock(t) +} + +func TestLitecoinRPC_GetTransaction(t *testing.T) { + rpcTest.TestGetTransaction(t) +} + +func TestLitecoinRPC_GetTransactionForMempool(t *testing.T) { + rpcTest.TestGetTransactionForMempool(t) +} + +func TestLitecoinRPC_MempoolSync(t *testing.T) { + rpcTest.TestMempoolSync(t) +} + +func TestLitecoinRPC_GetMempoolEntry(t *testing.T) { + rpcTest.TestGetMempoolEntry(t) +} + +func TestLitecoinRPC_EstimateSmartFee(t *testing.T) { + rpcTest.TestEstimateSmartFee(t) +} + +func TestLitecoinRPC_EstimateFee(t *testing.T) { + rpcTest.TestEstimateFee(t) +} diff --git a/bchain/tests/rpc/config.json b/bchain/tests/rpc/config.json index 8bbe587e..2e8be7a8 100644 --- a/bchain/tests/rpc/config.json +++ b/bchain/tests/rpc/config.json @@ -19,11 +19,21 @@ "user": "rpc", "pass": "rpc" }, + "Litecoin": { + "url": "http://localhost:8034", + "user": "rpc", + "pass": "rpc" + }, "Ethereum": { "url": "ws://localhost:8036", "user": null, "pass": null }, + "Dogecoin": { + "url": "http://localhost:8038", + "user": "rpc", + "pass": "rpcp" + }, "Namecoin": { "url": "http://localhost:8039", "user": "rpc", diff --git a/bchain/tests/rpc/testdata/Dogecoin.json b/bchain/tests/rpc/testdata/Dogecoin.json new file mode 100644 index 00000000..75b32f85 --- /dev/null +++ b/bchain/tests/rpc/testdata/Dogecoin.json @@ -0,0 +1,53 @@ +{ + "blockHeight": 2000002, + "blockHash": "6db0b6fc543cacbc4244f5fab23af56792a87eacc3149957a1922fd59d4d03e0", + "blockTime": 1512601001, + "blockTxs": [ + "cc3c92a9da8e28f18faf17efe74e96f541f03e3a300c55fef0c4dbc9e5b14c01", + "cb3cb43e34385556c9617699687b15f3dbad57cb0e8548114b66c55512c51a52", + "f6be02faa646a1ad764e4eb49fb6f02cbae69e5bc396a8977e59c7d887aa38b2", + "79883da5f57c942feb922e4bb635af631219aae2e70037459b07840945d6466c", + "23b5caddcb3c1222a0b97ff6f67185266b982465ef1a3546c7fc6ad8414ba7d0", + "82cf9f6822362ecfc0d47cc6b7372862ef0fdd14b259c42c8f2def8589f266f0", + "12a4e4daa6e3b842a2395c315f296f284ac2085e2f2fb4f356f94e08117963ba", + "cc136353be3db35e051e5dd34ff29f9cfe47a7b3f6ab7383fd644bc00ab1e348", + "f2db6749bc5f7e727f9e6c9a5553ad44dc59d9994dcadc24a57cf6042d1d07e7" + ], + "txDetails": { + "f6be02faa646a1ad764e4eb49fb6f02cbae69e5bc396a8977e59c7d887aa38b2": { + "hex": "0100000002361742ebea8bdce921359dbe528b23cc8050dc86950fd07011a74258cd0215dde5030000db00483045022100885238254408979e8f44c087c06333376980a73594c4d51c8cdf457934514e34022026287952d554f9f9f24c0a47096cce0db53547099aa700a91fa9046eb5c4960801483045022100a4ce7700bdf654dfe2a4fc5796bd3466be6af3691dc263821d4c71f91f9d9d1e02200b988bea6bf9e7d44e9a4b8cee9e390c72d284e3df1aacf6ab23368560b3e53e0147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52aeffffffff85f2900e642720043fae056075b48ea83cbd3653797c40fb4e7e512938dcda7573020000da004730440220138a8dd272f3081e54b172fb3e6cf43fe5375369d7e93c310ceba852c22ba16f022050de068f932f6d6db9889a8d4e489ffeda2ed1848cd2a71a06d635e6d1799b8e01483045022100af99d412c97062219ef56c3c8ad99023feab94042d310f792392940c61df9bdd02200b4542395624412260c96c46427509c2b801df33e15da08c6c505594c7c142010147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52aeffffffff01c0ebf7b90100000017a9147541523df4d0d0875c024e1906b0d195abaf20958700000000", + "txid": "f6be02faa646a1ad764e4eb49fb6f02cbae69e5bc396a8977e59c7d887aa38b2", + "blocktime": 1512601001, + "time": 1512601001, + "locktime": 0, + "version": 1, + "vin": [ + { + "txid": "dd1502cd5842a71170d00f9586dc5080cc238b52be9d3521e9dc8beaeb421736", + "vout": 997, + "scriptSig": { + "hex": "00483045022100885238254408979e8f44c087c06333376980a73594c4d51c8cdf457934514e34022026287952d554f9f9f24c0a47096cce0db53547099aa700a91fa9046eb5c4960801483045022100a4ce7700bdf654dfe2a4fc5796bd3466be6af3691dc263821d4c71f91f9d9d1e02200b988bea6bf9e7d44e9a4b8cee9e390c72d284e3df1aacf6ab23368560b3e53e0147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52ae" + }, + "sequence": 4294967295 + }, + { + "txid": "75dadc3829517e4efb407c795336bd3ca88eb4756005ae3f042027640e90f285", + "vout": 627, + "scriptSig": { + "hex": "004730440220138a8dd272f3081e54b172fb3e6cf43fe5375369d7e93c310ceba852c22ba16f022050de068f932f6d6db9889a8d4e489ffeda2ed1848cd2a71a06d635e6d1799b8e01483045022100af99d412c97062219ef56c3c8ad99023feab94042d310f792392940c61df9bdd02200b4542395624412260c96c46427509c2b801df33e15da08c6c505594c7c142010147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52ae" + }, + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 74.15000000, + "n": 0, + "scriptPubKey": { + "hex": "a9147541523df4d0d0875c024e1906b0d195abaf209587" + } + } + ] + } + } +} \ No newline at end of file diff --git a/bchain/tests/rpc/testdata/Litecoin.json b/bchain/tests/rpc/testdata/Litecoin.json new file mode 100644 index 00000000..4f5a5f5a --- /dev/null +++ b/bchain/tests/rpc/testdata/Litecoin.json @@ -0,0 +1,49 @@ +{ + "blockHeight": 1377592, + "blockHash": "bddb1cfbd474e9516399b373e411bd33c1a71cb01aa8469a27d397ef0a891c7d", + "blockTime": 1519947864, + "blockTxs": [ + "84e9147bf6e171adbda3b3961e467652286d9d9c2933d19326bf84766d047922", + "8d6e628b891dd17bfe3bb5a24a6c7f02ebc2cf499a85515d0325033aa74ab53a", + "5b77ca9735f65d110b086be410658d0239e1fcee13231942a262b35a5b8d6a91", + "19ad3daa2447be4e000d822f79ce252f274b016dacf1418b433d36c0aaf24f18", + "5bffbf0c8ff66d298d94dc323c3644e21932dfc733603d6637ff46cb8d34466c", + "90d587e35b23905f0125111f41d69bb6c7eed44f0944caad2903aae1f174ac49" + ], + "txDetails": { + "19ad3daa2447be4e000d822f79ce252f274b016dacf1418b433d36c0aaf24f18": { + "hex": "010000000001011f9216c16c78386540d7ae7d32657c388ef5f204596f84ea0851dcb78c479a87010000001716001432d094c8e2efd308d2c69affd6712ebbf7a5a286ffffffff0289544a110000000017a91457ca840d6c811dd6808722babe3f88d2fdb2ca14875bb1c9180000000017a91482696a9b4188eda8f93b120315cad4260cbb90db8702483045022100aa256153317133fa719180935017671e33ca77df6f5426554f5b0855f07a392b02202684e8c30623e1c4e753ea23a95bd571e1fda7d15dc0b1e2d54ff5bc50329256012103399a1d98f0733ef400ff8d4f43fe4543065f7f387c863361c77a8826321ca6fb00000000", + "txid": "19ad3daa2447be4e000d822f79ce252f274b016dacf1418b433d36c0aaf24f18", + "blocktime": 1519947864, + "time": 1519947864, + "locktime": 0, + "version": 1, + "vin": [ + { + "txid": "879a478cb7dc5108ea846f5904f2f58e387c65327daed7406538786cc116921f", + "vout": 1, + "scriptSig": { + "hex": "16001432d094c8e2efd308d2c69affd6712ebbf7a5a286" + }, + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 2.90083977, + "n": 0, + "scriptPubKey": { + "hex": "a91457ca840d6c811dd6808722babe3f88d2fdb2ca1487" + } + }, + { + "value": 4.15871323, + "n": 1, + "scriptPubKey": { + "hex": "a91482696a9b4188eda8f93b120315cad4260cbb90db87" + } + } + ] + } + } +} \ No newline at end of file From d58c620d85fd2432090a3e16f62dd9616d64cd14 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 29 Aug 2018 21:37:06 +0200 Subject: [PATCH 041/123] Stabilize dogecoin rpc test --- bchain/coins/dogecoin/dogecoinrpc_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bchain/coins/dogecoin/dogecoinrpc_test.go b/bchain/coins/dogecoin/dogecoinrpc_test.go index 17ffb0ef..b9d3bbc4 100644 --- a/bchain/coins/dogecoin/dogecoinrpc_test.go +++ b/bchain/coins/dogecoin/dogecoinrpc_test.go @@ -59,14 +59,12 @@ func TestDogecoinRPC_MempoolSync(t *testing.T) { rpcTest.TestMempoolSync(t) } -func TestDogecoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - func TestDogecoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) + t.Skip("skipping test, unreliable") + // rpcTest.TestEstimateSmartFee(t) } func TestDogecoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) + t.Skip("skipping test, unreliable") + // rpcTest.TestEstimateFee(t) } From 2f0e4e2a0ff67cb7d78af3ab72a37e536d546074 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 30 Aug 2018 13:06:11 +0200 Subject: [PATCH 042/123] Fix test tags --- bchain/coins/btc/bitcoinparser_test.go | 2 +- server/socketio_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 4b4308a1..a9d367fa 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -1,4 +1,4 @@ -// build unittest +// +build unittest package btc diff --git a/server/socketio_test.go b/server/socketio_test.go index 108815cd..09f2d368 100644 --- a/server/socketio_test.go +++ b/server/socketio_test.go @@ -1,4 +1,4 @@ -// +build unittest +// +build integration package server From af6de32165f24397fd747187d977185520122bdc Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 30 Aug 2018 13:06:49 +0200 Subject: [PATCH 043/123] Use indexv3 in socketio getDetailedTransaction --- server/socketio.go | 48 +++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/server/socketio.go b/server/socketio.go index 9d36f2e4..d6aa5c76 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -237,11 +237,11 @@ type txInputs struct { // ScriptAsm *string `json:"scriptAsm"` Sequence int64 `json:"sequence"` Address *string `json:"address"` - Satoshis string `json:"satoshis"` + Satoshis uint64 `json:"satoshis"` } type txOutputs struct { - Satoshis string `json:"satoshis"` + Satoshis uint64 `json:"satoshis"` Script *string `json:"script"` // ScriptAsm *string `json:"scriptAsm"` SpentTxID *string `json:"spentTxId,omitempty"` @@ -260,15 +260,15 @@ type resTx struct { Locktime int `json:"locktime,omitempty"` // Size int `json:"size,omitempty"` Inputs []txInputs `json:"inputs"` - // InputSatoshis int64 `json:"inputSatoshis,omitempty"` + // InputSatoshis uint64 `json:"inputSatoshis,omitempty"` Outputs []txOutputs `json:"outputs"` - // OutputSatoshis int64 `json:"outputSatoshis,omitempty"` - // FeeSatoshis int64 `json:"feeSatoshis,omitempty"` + // OutputSatoshis uint64 `json:"outputSatoshis,omitempty"` + // FeeSatoshis uint64 `json:"feeSatoshis,omitempty"` } type addressHistoryItem struct { Addresses map[string]addressHistoryIndexes `json:"addresses"` - Satoshis string `json:"satoshis"` + Satoshis uint64 `json:"satoshis"` Confirmations int `json:"confirmations"` Tx resTx `json:"tx"` } @@ -353,7 +353,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r for _, vout := range tx.Vout { aoh := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: vout.ValueSat.String(), + Satoshis: vout.ValueSat.Uint64(), Script: &aoh, } voutAddr, err := s.getAddressesFromVout(&vout) @@ -608,20 +608,38 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai OutputIndex: int(vin.Vout), } if vin.Txid != "" { - otx, _, err := s.txCache.GetTransaction(vin.Txid, bestheight) + var voutAddr []string + // load spending addresses from TxAddresses + ta, err := s.db.GetTxAddresses(vin.Txid) if err != nil { return res, err } - if len(otx.Vout) > int(vin.Vout) { - vout := &otx.Vout[vin.Vout] - voutAddr, err := s.getAddressesFromVout(vout) + if ta == nil { + // the tx may be in mempool, try to load it from backend + otx, _, err := s.txCache.GetTransaction(vin.Txid, bestheight) if err != nil { return res, err } - if len(voutAddr) > 0 { - ai.Address = &voutAddr[0] + if len(otx.Vout) > int(vin.Vout) { + vout := &otx.Vout[vin.Vout] + voutAddr, err = s.getAddressesFromVout(vout) + if err != nil { + return res, err + } + ai.Satoshis = vout.ValueSat.Uint64() } - ai.Satoshis = vout.ValueSat.String() + } else { + if len(ta.Outputs) > int(vin.Vout) { + output := &ta.Outputs[vin.Vout] + ai.Satoshis = output.ValueSat.Uint64() + voutAddr, _, err = output.Addresses(s.chainParser) + if err != nil { + return res, err + } + } + } + if len(voutAddr) > 0 { + ai.Address = &voutAddr[0] } } hi = append(hi, ai) @@ -629,7 +647,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai for _, vout := range tx.Vout { aos := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: vout.ValueSat.String(), + Satoshis: vout.ValueSat.Uint64(), Script: &aos, } voutAddr, err := s.getAddressesFromVout(&vout) From 82c64ae947d360d34a17ab5e2454309ab2dc8645 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 30 Aug 2018 13:19:50 +0200 Subject: [PATCH 044/123] Make satoshi amounts signed in socketio --- server/socketio.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/socketio.go b/server/socketio.go index d6aa5c76..f9924452 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -237,11 +237,11 @@ type txInputs struct { // ScriptAsm *string `json:"scriptAsm"` Sequence int64 `json:"sequence"` Address *string `json:"address"` - Satoshis uint64 `json:"satoshis"` + Satoshis int64 `json:"satoshis"` } type txOutputs struct { - Satoshis uint64 `json:"satoshis"` + Satoshis int64 `json:"satoshis"` Script *string `json:"script"` // ScriptAsm *string `json:"scriptAsm"` SpentTxID *string `json:"spentTxId,omitempty"` @@ -260,15 +260,15 @@ type resTx struct { Locktime int `json:"locktime,omitempty"` // Size int `json:"size,omitempty"` Inputs []txInputs `json:"inputs"` - // InputSatoshis uint64 `json:"inputSatoshis,omitempty"` + // InputSatoshis int64 `json:"inputSatoshis,omitempty"` Outputs []txOutputs `json:"outputs"` - // OutputSatoshis uint64 `json:"outputSatoshis,omitempty"` - // FeeSatoshis uint64 `json:"feeSatoshis,omitempty"` + // OutputSatoshis int64 `json:"outputSatoshis,omitempty"` + // FeeSatoshis int64 `json:"feeSatoshis,omitempty"` } type addressHistoryItem struct { Addresses map[string]addressHistoryIndexes `json:"addresses"` - Satoshis uint64 `json:"satoshis"` + Satoshis int64 `json:"satoshis"` Confirmations int `json:"confirmations"` Tx resTx `json:"tx"` } @@ -353,7 +353,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r for _, vout := range tx.Vout { aoh := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: vout.ValueSat.Uint64(), + Satoshis: vout.ValueSat.Int64(), Script: &aoh, } voutAddr, err := s.getAddressesFromVout(&vout) @@ -626,12 +626,12 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai if err != nil { return res, err } - ai.Satoshis = vout.ValueSat.Uint64() + ai.Satoshis = vout.ValueSat.Int64() } } else { if len(ta.Outputs) > int(vin.Vout) { output := &ta.Outputs[vin.Vout] - ai.Satoshis = output.ValueSat.Uint64() + ai.Satoshis = output.ValueSat.Int64() voutAddr, _, err = output.Addresses(s.chainParser) if err != nil { return res, err @@ -647,7 +647,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai for _, vout := range tx.Vout { aos := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: vout.ValueSat.Uint64(), + Satoshis: vout.ValueSat.Int64(), Script: &aos, } voutAddr, err := s.getAddressesFromVout(&vout) From 1a931d8a65451c90a852675f934e744ccaf3c173 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 30 Aug 2018 15:25:03 +0200 Subject: [PATCH 045/123] Implement view of OP_RETURN scripts --- api/worker.go | 3 +-- bchain/coins/btc/bitcoinparser.go | 23 ++++++++++++++++++++++- bchain/coins/btc/bitcoinparser_test.go | 12 +++++++++--- static/templates/txdetail.html | 8 +++----- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/api/worker.go b/api/worker.go index 397b1247..05d0c55a 100644 --- a/api/worker.go +++ b/api/worker.go @@ -112,8 +112,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex - - vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses + vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = w.getAddressesFromVout(bchainVout) if spendingTxs { // TODO } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index f741adf9..e48829fd 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -63,7 +63,7 @@ func (p *BitcoinParser) GetAddrDescFromAddress(address string) ([]byte, error) { // GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable func (p *BitcoinParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { - return p.outputScriptToAddresses(addrDesc) + return p.OutputScriptToAddressesFunc(addrDesc) } // GetScriptFromAddrDesc returns output script for given address descriptor @@ -97,6 +97,27 @@ func (p *BitcoinParser) outputScriptToAddresses(script []byte) ([]string, bool, var s bool if sc != txscript.NonStandardTy && sc != txscript.NullDataTy { s = true + } else { + if len(script) > 1 && script[0] == txscript.OP_RETURN && len(rv) == 0 { + l := int(script[1]) + data := script[2:] + if l == len(data) { + isASCII := true + for _, c := range data { + if c < 32 || c > 127 { + isASCII = false + break + } + } + var ed string + if isASCII { + ed = "(" + string(data) + ")" + } else { + ed = hex.EncodeToString([]byte{byte(l)}) + " " + hex.EncodeToString(data) + } + rv = []string{"OP_RETURN " + ed} + } + } } return rv, s, nil } diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index a9d367fa..74f7b4cb 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -102,10 +102,16 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { wantErr: false, }, { - // TODO handle OP_RETURN better - name: "OP_RETURN", + name: "OP_RETURN ascii", args: args{script: "6a0461686f6a"}, - want: []string{}, + want: []string{"OP_RETURN (ahoj)"}, + want2: false, + wantErr: false, + }, + { + name: "OP_RETURN hex", + args: args{script: "6a072020f1686f6a20"}, + want: []string{"OP_RETURN 07 2020f1686f6a20"}, want2: false, wantErr: false, }, diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 2c70c912..f352700c 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -18,12 +18,11 @@ {{range $a := $vin.Addresses}} - {{if eq $a $addr}}{{$a}}{{else}} - {{$a}}{{end}} + {{if and (ne $a $addr) $vin.Searchable}}{{$a}}{{else}}{{$a}}{{end}} {{else}} No Inputs (Newly Generated Coins) - {{end}} {{if $vin.Addresses}} + {{end}}{{if $vin.Addresses}} {{formatAmount $vin.Value}} {{$cs}} {{end}} @@ -47,8 +46,7 @@ {{range $a := $vout.ScriptPubKey.Addresses}} - {{if eq $a $addr}}{{$a}}{{else}} - {{$a}}{{end}} + {{if and (ne $a $addr) $vout.ScriptPubKey.Searchable}}{{$a}}{{else}}{{$a}}{{end}} {{else}} Unparsed address From 676aabfaa68aa7cb4c9ffa2fd171d124bda08bc3 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 30 Aug 2018 22:39:03 +0200 Subject: [PATCH 046/123] Make AddressDescriptor as type,expose DB/mempool methods working with it --- bchain/coins/bch/bcashparser.go | 2 +- bchain/coins/blockchain.go | 5 + bchain/coins/btc/bitcoinparser.go | 8 +- bchain/coins/btc/bitcoinrpc.go | 9 +- bchain/coins/eth/ethparser.go | 8 +- bchain/coins/eth/ethrpc.go | 12 ++- bchain/coins/monacoin/monacoinparser.go | 5 +- bchain/coins/monacoin/monacoinparser_test.go | 75 +++++++++++++++ bchain/coins/zec/zcashparser.go | 8 +- bchain/mempool_nonutxo.go | 7 +- bchain/mempool_utxo.go | 5 + bchain/types.go | 12 ++- db/rocksdb.go | 99 +++++++++++--------- db/rocksdb_test.go | 36 +++---- 14 files changed, 206 insertions(+), 85 deletions(-) diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index ccfebc29..2c804867 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -79,7 +79,7 @@ func GetChainParams(chain string) *chaincfg.Params { } // GetAddrDescFromAddress returns internal address representation of given address -func (p *BCashParser) GetAddrDescFromAddress(address string) ([]byte, error) { +func (p *BCashParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { return p.addressToOutputScript(address) } diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index ddd1ffd2..b0be1d2b 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -205,6 +205,11 @@ func (c *blockChainWithMetrics) GetMempoolTransactions(address string) (v []stri return c.b.GetMempoolTransactions(address) } +func (c *blockChainWithMetrics) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []string, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactionsForAddrDesc", s, err) }(time.Now()) + return c.b.GetMempoolTransactionsForAddrDesc(addrDesc) +} + func (c *blockChainWithMetrics) GetMempoolEntry(txid string) (v *bchain.MempoolEntry, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempoolEntry", s, err) }(time.Now()) return c.b.GetMempoolEntry(txid) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index e48829fd..a4745be0 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -52,22 +52,22 @@ func GetChainParams(chain string) *chaincfg.Params { } // GetAddrDescFromVout returns internal address representation (descriptor) of given transaction output -func (p *BitcoinParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { +func (p *BitcoinParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { return hex.DecodeString(output.ScriptPubKey.Hex) } // GetAddrDescFromAddress returns internal address representation (descriptor) of given address -func (p *BitcoinParser) GetAddrDescFromAddress(address string) ([]byte, error) { +func (p *BitcoinParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { return p.addressToOutputScript(address) } // GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable -func (p *BitcoinParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { +func (p *BitcoinParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { return p.OutputScriptToAddressesFunc(addrDesc) } // GetScriptFromAddrDesc returns output script for given address descriptor -func (p *BitcoinParser) GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) { +func (p *BitcoinParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]byte, error) { return addrDesc, nil } diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index a6305106..ead2380c 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -617,12 +617,17 @@ func (b *BitcoinRPC) ResyncMempool(onNewTxAddr func(txid string, addr string)) ( return b.Mempool.Resync(onNewTxAddr) } -// GetMempoolTransactions returns slice of mempool transactions for given address. +// GetMempoolTransactions returns slice of mempool transactions for given address func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]string, error) { return b.Mempool.GetTransactions(address) } -// EstimateSmartFee returns fee estimation. +// GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor +func (b *BitcoinRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, error) { + return b.Mempool.GetAddrDescTransactions(addrDesc) +} + +// EstimateSmartFee returns fee estimation func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { // use EstimateFee if EstimateSmartFee is not supported if !b.ChainConfig.SupportsEstimateSmartFee && b.ChainConfig.SupportsEstimateFee { diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index f933b761..d76aa6bf 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -119,7 +119,7 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } // GetAddrDescFromVout returns internal address representation of given transaction output -func (p *EthereumParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { +func (p *EthereumParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { if len(output.ScriptPubKey.Addresses) != 1 { return nil, bchain.ErrAddressMissing } @@ -131,7 +131,7 @@ func has0xPrefix(s string) bool { } // GetAddrDescFromAddress returns internal address representation of given address -func (p *EthereumParser) GetAddrDescFromAddress(address string) ([]byte, error) { +func (p *EthereumParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { // github.com/ethereum/go-ethereum/common.HexToAddress does not handle address errors, using own decoding if has0xPrefix(address) { address = address[2:] @@ -146,12 +146,12 @@ func (p *EthereumParser) GetAddrDescFromAddress(address string) ([]byte, error) } // GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable -func (p *EthereumParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { +func (p *EthereumParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { return []string{hexutil.Encode(addrDesc)}, true, nil } // GetScriptFromAddrDesc returns output script for given address descriptor -func (p *EthereumParser) GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) { +func (p *EthereumParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]byte, error) { return addrDesc, nil } diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 2ef90b5d..cb08b664 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -480,12 +480,12 @@ func (b *EthereumRPC) GetMempool() ([]string, error) { return body.Transactions, nil } -// EstimateFee returns fee estimation. +// EstimateFee returns fee estimation func (b *EthereumRPC) EstimateFee(blocks int) (big.Int, error) { return b.EstimateSmartFee(blocks, true) } -// EstimateSmartFee returns fee estimation. +// EstimateSmartFee returns fee estimation func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() @@ -503,7 +503,7 @@ func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, return r, nil } -// SendRawTransaction sends raw transaction. +// SendRawTransaction sends raw transaction func (b *EthereumRPC) SendRawTransaction(tx string) (string, error) { return "", errors.New("SendRawTransaction: not implemented") } @@ -512,10 +512,16 @@ func (b *EthereumRPC) ResyncMempool(onNewTxAddr func(txid string, addr string)) return b.Mempool.Resync(onNewTxAddr) } +// GetMempoolTransactions returns slice of mempool transactions for given address func (b *EthereumRPC) GetMempoolTransactions(address string) ([]string, error) { return b.Mempool.GetTransactions(address) } +// GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor +func (b *EthereumRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, error) { + return b.Mempool.GetAddrDescTransactions(addrDesc) +} + func (b *EthereumRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { return nil, errors.New("GetMempoolEntry: not implemented") } diff --git a/bchain/coins/monacoin/monacoinparser.go b/bchain/coins/monacoin/monacoinparser.go index 88d0870c..4a05bdf0 100644 --- a/bchain/coins/monacoin/monacoinparser.go +++ b/bchain/coins/monacoin/monacoinparser.go @@ -1,6 +1,7 @@ package monacoin import ( + "blockbook/bchain" "blockbook/bchain/coins/btc" "github.com/btcsuite/btcd/chaincfg" @@ -89,8 +90,8 @@ func GetMonaChainParams(chain string) *monacoinCfg.Params { } } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *MonacoinParser) GetAddrIDFromAddress(address string) ([]byte, error) { +// GetAddrDescFromAddress returns internal address representation (descriptor) of given address +func (p *MonacoinParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { return p.addressToOutputScript(address) } diff --git a/bchain/coins/monacoin/monacoinparser_test.go b/bchain/coins/monacoin/monacoinparser_test.go index 9f559749..3b9e759e 100644 --- a/bchain/coins/monacoin/monacoinparser_test.go +++ b/bchain/coins/monacoin/monacoinparser_test.go @@ -121,6 +121,81 @@ func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { } } +func Test_GetAddressesFromAddrDesc(t *testing.T) { + type args struct { + script string + } + tests := []struct { + name string + args args + want []string + want2 bool + wantErr bool + }{ + { + name: "P2PKH", + args: args{script: "76a91451dadacc7021440cbe4ca148a5db563b329b4c0388ac"}, + want: []string{"MFMy9FwJsV6HiN5eZDqDETw4pw52q3UGrb"}, + want2: true, + wantErr: false, + }, + { + name: "P2SH", + args: args{script: "a9146449f568c9cd2378138f2636e1567112a184a9e887"}, + want: []string{"PHjTKtgYLTJ9D2Bzw2f6xBB41KBm2HeGfg"}, + want2: true, + wantErr: false, + }, + { + name: "P2WPKH", + args: args{script: "0014a96d3cef194f469b33801f868ec9bc89a8831c22"}, + want: []string{"mona1q49knemcefarfkvuqr7rgajdu3x5gx8pzdnurgq"}, + want2: true, + wantErr: false, + }, + { + name: "P2WSH", + args: args{script: "002009d27aa88e70cb7a0da620908c9bc08ac6c633bd1a61036312e514396aeb4893"}, + want: []string{"mona1qp8f842ywwr9h5rdxyzggex7q3trvvvaarfssxccju52rj6htfzfsqr79j2"}, + want2: true, + wantErr: false, + }, + { + name: "OP_RETURN ascii", + args: args{script: "6a0461686f6a"}, + want: []string{"OP_RETURN (ahoj)"}, + want2: false, + wantErr: false, + }, + { + name: "OP_RETURN hex", + args: args{script: "6a072020f1686f6a20"}, + want: []string{"OP_RETURN 07 2020f1686f6a20"}, + want2: false, + wantErr: false, + }, + } + + parser := NewMonacoinParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.script) + got, got2, err := parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("outputScriptToAddresses() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) + } + }) + } +} + var ( testTx1 bchain.Tx diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index 84f422db..7b7e6e20 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -52,7 +52,7 @@ func GetChainParams(chain string) *chaincfg.Params { } // GetAddrDescFromVout returns internal address representation of given transaction output -func (p *ZCashParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { +func (p *ZCashParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { if len(output.ScriptPubKey.Addresses) != 1 { return nil, nil } @@ -61,19 +61,19 @@ func (p *ZCashParser) GetAddrDescFromVout(output *bchain.Vout) ([]byte, error) { } // GetAddrDescFromAddress returns internal address representation of given address -func (p *ZCashParser) GetAddrDescFromAddress(address string) ([]byte, error) { +func (p *ZCashParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { hash, _, err := utils.CheckDecode(address) return hash, err } // GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable -func (p *ZCashParser) GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) { +func (p *ZCashParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { // TODO implement return nil, false, errors.New("GetAddressesFromAddrDesc: not implemented") } // GetScriptFromAddrDesc returns output script for given address descriptor -func (p *ZCashParser) GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) { +func (p *ZCashParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]byte, error) { // TODO implement return nil, errors.New("GetScriptFromAddrDesc: not implemented") } diff --git a/bchain/mempool_nonutxo.go b/bchain/mempool_nonutxo.go index 3d1aa66f..1930aaa6 100644 --- a/bchain/mempool_nonutxo.go +++ b/bchain/mempool_nonutxo.go @@ -12,7 +12,7 @@ type NonUTXOMempool struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrDescToTx map[string][]outpoint + addrDescToTx map[string][]outpoint } // NewNonUTXOMempool creates new mempool handler. @@ -27,6 +27,11 @@ func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { if err != nil { return nil, err } + return m.GetAddrDescTransactions(addrDesc) +} + +// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor +func (m *NonUTXOMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]string, error) { m.mux.Lock() defer m.mux.Unlock() outpoints := m.addrDescToTx[string(addrDesc)] diff --git a/bchain/mempool_utxo.go b/bchain/mempool_utxo.go index f89daa29..f111b513 100644 --- a/bchain/mempool_utxo.go +++ b/bchain/mempool_utxo.go @@ -74,6 +74,11 @@ func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { if err != nil { return nil, err } + return m.GetAddrDescTransactions(addrDesc) +} + +// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor +func (m *UTXOMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]string, error) { m.mux.Lock() defer m.mux.Unlock() outpoints := m.addrDescToTx[string(addrDesc)] diff --git a/bchain/types.go b/bchain/types.go index 50d69f9a..e27bca57 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -111,6 +111,9 @@ func (e *RPCError) Error() string { return fmt.Sprintf("%d: %s", e.Code, e.Message) } +// AddressDescriptor is an opaque type obtained by parser.GetAddrDesc* methods +type AddressDescriptor []byte + // BlockChain defines common interface to block chain daemon type BlockChain interface { // life-cycle methods @@ -137,6 +140,7 @@ type BlockChain interface { // mempool ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) GetMempoolTransactions(address string) ([]string, error) + GetMempoolTransactionsForAddrDesc(addrDesc AddressDescriptor) ([]string, error) GetMempoolEntry(txid string) (*MempoolEntry, error) // parser GetChainParser() BlockChainParser @@ -158,10 +162,10 @@ type BlockChainParser interface { // it uses string operations to avoid problems with rounding AmountToBigInt(n json.Number) (big.Int, error) // address descriptor conversions - GetAddrDescFromVout(output *Vout) ([]byte, error) - GetAddrDescFromAddress(address string) ([]byte, error) - GetAddressesFromAddrDesc(addrDesc []byte) ([]string, bool, error) - GetScriptFromAddrDesc(addrDesc []byte) ([]byte, error) + GetAddrDescFromVout(output *Vout) (AddressDescriptor, error) + GetAddrDescFromAddress(address string) (AddressDescriptor, error) + GetAddressesFromAddrDesc(addrDesc AddressDescriptor) ([]string, bool, error) + GetScriptFromAddrDesc(addrDesc AddressDescriptor) ([]byte, error) // transactions PackedTxidLen() int PackTxid(txid string) ([]byte, error) diff --git a/db/rocksdb.go b/db/rocksdb.go index 82c7d46d..00967f0d 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -155,6 +155,13 @@ func (d *RocksDB) GetMemoryStats() string { return fmt.Sprintf("%+v", m) } +// StopIteration is returned by callback function to signal stop of iteration +type StopIteration struct{} + +func (e *StopIteration) Error() string { + return "" +} + // GetTransactions finds all input/output transactions for address // Transaction are passed to callback function. func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { @@ -165,7 +172,12 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f if err != nil { return err } + return d.GetAddrDescTransactions(addrDesc, lower, higher, fn) +} +// GetAddrDescTransactions finds all input/output transactions for address descriptor +// Transaction are passed to callback function. +func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { kstart := packAddressKey(addrDesc, lower) kstop := packAddressKey(addrDesc, higher) @@ -200,6 +212,9 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f return err } if err := fn(tx, vout, isOutput); err != nil { + if _, ok := err.(*StopIteration); ok { + return nil + } return err } } @@ -281,22 +296,22 @@ type outpoint struct { } type TxInput struct { - addrDesc []byte + AddrDesc bchain.AddressDescriptor ValueSat big.Int } func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { - return p.GetAddressesFromAddrDesc(ti.addrDesc) + return p.GetAddressesFromAddrDesc(ti.AddrDesc) } type TxOutput struct { - addrDesc []byte + AddrDesc bchain.AddressDescriptor Spent bool ValueSat big.Int } func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { - return p.GetAddressesFromAddrDesc(to.addrDesc) + return p.GetAddressesFromAddrDesc(to.AddrDesc) } type TxAddresses struct { @@ -322,7 +337,7 @@ type blockTxs struct { inputs []outpoint } -func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc []byte, logText string) { +func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc bchain.AddressDescriptor, logText string) { ad, _, err := d.chainParser.GetAddressesFromAddrDesc(addrDesc) had := hex.EncodeToString(addrDesc) if err != nil { @@ -363,7 +378,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string } continue } - tao.addrDesc = addrDesc + tao.AddrDesc = addrDesc strAddrDesc := string(addrDesc) // check that the address was used already in this block o, processed := addresses[strAddrDesc] @@ -377,7 +392,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string }) ab, e := balances[strAddrDesc] if !e { - ab, err = d.getAddrDescBalance(addrDesc) + ab, err = d.GetAddrDescBalance(addrDesc) if err != nil { return err } @@ -431,18 +446,18 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string if ot.Spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } - tai.addrDesc = ot.addrDesc + tai.AddrDesc = ot.AddrDesc tai.ValueSat = ot.ValueSat // mark the output as spent in tx ot.Spent = true - if len(ot.addrDesc) == 0 { + if len(ot.AddrDesc) == 0 { if !logged { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) logged = true } continue } - strAddrDesc := string(ot.addrDesc) + strAddrDesc := string(ot.AddrDesc) // check that the address was used already in this block o, processed := addresses[strAddrDesc] if processed { @@ -455,7 +470,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string }) ab, e := balances[strAddrDesc] if !e { - ab, err = d.getAddrDescBalance(ot.addrDesc) + ab, err = d.GetAddrDescBalance(ot.AddrDesc) if err != nil { return err } @@ -470,7 +485,7 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string } ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat) if ab.BalanceSat.Sign() < 0 { - d.resetValueSatToZero(&ab.BalanceSat, ot.addrDesc, "balance") + d.resetValueSatToZero(&ab.BalanceSat, ot.AddrDesc, "balance") } ab.SentSat.Add(&ab.SentSat, &ot.ValueSat) } @@ -489,7 +504,7 @@ func processedInTx(o []outpoint, btxID []byte) bool { func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error { for addrDesc, outpoints := range addresses { - ba := []byte(addrDesc) + ba := bchain.AddressDescriptor(addrDesc) key := packAddressKey(ba, height) val := d.packOutpoints(outpoints) wb.PutCF(d.cfh[cfAddresses], key, val) @@ -513,14 +528,14 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBa for addrDesc, ab := range abm { // balance with 0 transactions is removed from db - happens in disconnect if ab == nil || ab.Txs <= 0 { - wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrDesc)) + wb.DeleteCF(d.cfh[cfAddressBalance], bchain.AddressDescriptor(addrDesc)) } else { l := packVaruint(uint(ab.Txs), buf) ll := packBigint(&ab.SentSat, buf[l:]) l += ll ll = packBigint(&ab.BalanceSat, buf[l:]) l += ll - wb.PutCF(d.cfh[cfAddressBalance], []byte(addrDesc), buf[:l]) + wb.PutCF(d.cfh[cfAddressBalance], bchain.AddressDescriptor(addrDesc), buf[:l]) } } return nil @@ -609,7 +624,7 @@ func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { return bt, nil } -func (d *RocksDB) getAddrDescBalance(addrDesc []byte) (*AddrBalance, error) { +func (d *RocksDB) GetAddrDescBalance(addrDesc bchain.AddressDescriptor) (*AddrBalance, error) { val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrDesc) if err != nil { return nil, err @@ -636,7 +651,7 @@ func (d *RocksDB) GetAddressBalance(address string) (*AddrBalance, error) { if err != nil { return nil, err } - return d.getAddrDescBalance(addrDesc) + return d.GetAddrDescBalance(addrDesc) } func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { @@ -680,23 +695,23 @@ func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { } func appendTxInput(txi *TxInput, buf []byte, varBuf []byte) []byte { - la := len(txi.addrDesc) + la := len(txi.AddrDesc) l := packVaruint(uint(la), varBuf) buf = append(buf, varBuf[:l]...) - buf = append(buf, txi.addrDesc...) + buf = append(buf, txi.AddrDesc...) l = packBigint(&txi.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf } func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { - la := len(txo.addrDesc) + la := len(txo.AddrDesc) if txo.Spent { la = ^la } l := packVarint(la, varBuf) buf = append(buf, varBuf[:l]...) - buf = append(buf, txo.addrDesc...) + buf = append(buf, txo.AddrDesc...) l = packBigint(&txo.ValueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf @@ -723,8 +738,8 @@ func unpackTxAddresses(buf []byte) (*TxAddresses, error) { func unpackTxInput(ti *TxInput, buf []byte) int { al, l := unpackVaruint(buf) - ti.addrDesc = make([]byte, al) - copy(ti.addrDesc, buf[l:l+int(al)]) + ti.AddrDesc = make([]byte, al) + copy(ti.AddrDesc, buf[l:l+int(al)]) al += uint(l) ti.ValueSat, l = unpackBigint(buf[al:]) return l + int(al) @@ -736,8 +751,8 @@ func unpackTxOutput(to *TxOutput, buf []byte) int { to.Spent = true al = ^al } - to.addrDesc = make([]byte, al) - copy(to.addrDesc, buf[l:l+al]) + to.AddrDesc = make([]byte, al) + copy(to.AddrDesc, buf[l:l+al]) al += l to.ValueSat, l = unpackBigint(buf[al:]) return l + al @@ -790,7 +805,7 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { return outpoints, p, nil } -func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc []byte, btxid []byte, vout int32, bh uint32) error { +func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc bchain.AddressDescriptor, btxid []byte, vout int32, bh uint32) error { if len(addrDesc) > 0 { if len(addrDesc) > maxAddrDescLen { glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc)) @@ -846,7 +861,7 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. } } for addrDesc, outpoints := range addresses { - key := packAddressKey([]byte(addrDesc), block.Height) + key := packAddressKey(bchain.AddressDescriptor(addrDesc), block.Height) switch op { case opInsert: val := d.packOutpoints(outpoints) @@ -1022,12 +1037,12 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *TxAddresses, txAddressesToUpdate map[string]*TxAddresses, balances map[string]*AddrBalance) error { addresses := make(map[string]struct{}) - getAddressBalance := func(addrDesc []byte) (*AddrBalance, error) { + getAddressBalance := func(addrDesc bchain.AddressDescriptor) (*AddrBalance, error) { var err error s := string(addrDesc) b, fb := balances[s] if !fb { - b, err = d.getAddrDescBalance(addrDesc) + b, err = d.GetAddrDescBalance(addrDesc) if err != nil { return nil, err } @@ -1036,13 +1051,13 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, return b, nil } for i, t := range txa.Inputs { - if len(t.addrDesc) > 0 { - s := string(t.addrDesc) + if len(t.AddrDesc) > 0 { + s := string(t.AddrDesc) _, exist := addresses[s] if !exist { addresses[s] = struct{}{} } - b, err := getAddressBalance(t.addrDesc) + b, err := getAddressBalance(t.AddrDesc) if err != nil { return err } @@ -1053,12 +1068,12 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } b.SentSat.Sub(&b.SentSat, &t.ValueSat) if b.SentSat.Sign() < 0 { - d.resetValueSatToZero(&b.SentSat, t.addrDesc, "sent amount") + d.resetValueSatToZero(&b.SentSat, t.AddrDesc, "sent amount") } b.BalanceSat.Add(&b.BalanceSat, &t.ValueSat) } else { - ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.addrDesc) - had := hex.EncodeToString(t.addrDesc) + ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) + had := hex.EncodeToString(t.AddrDesc) glog.Warningf("Balance for address %s (%s) not found", ad, had) } s = string(inputs[i].btxID) @@ -1074,13 +1089,13 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } } for _, t := range txa.Outputs { - if len(t.addrDesc) > 0 { - s := string(t.addrDesc) + if len(t.AddrDesc) > 0 { + s := string(t.AddrDesc) _, exist := addresses[s] if !exist { addresses[s] = struct{}{} } - b, err := getAddressBalance(t.addrDesc) + b, err := getAddressBalance(t.AddrDesc) if err != nil { return err } @@ -1091,11 +1106,11 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } b.BalanceSat.Sub(&b.BalanceSat, &t.ValueSat) if b.BalanceSat.Sign() < 0 { - d.resetValueSatToZero(&b.BalanceSat, t.addrDesc, "balance") + d.resetValueSatToZero(&b.BalanceSat, t.AddrDesc, "balance") } } else { - ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.addrDesc) - had := hex.EncodeToString(t.addrDesc) + ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) + had := hex.EncodeToString(t.AddrDesc) glog.Warningf("Balance for address %s (%s) not found", ad, had) } } @@ -1427,7 +1442,7 @@ func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) er // Helpers -func packAddressKey(addrDesc []byte, height uint32) []byte { +func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte { bheight := packUint(height) buf := make([]byte, 0, len(addrDesc)+len(bheight)) buf = append(buf, addrDesc...) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index e1785fda..5b04fa3f 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -816,22 +816,22 @@ func TestRocksDB_Index_UTXO(t *testing.T) { Height: 225494, Inputs: []TxInput{ { - addrDesc: addressToAddrDesc(addr3, d.chainParser), + AddrDesc: addressToAddrDesc(addr3, d.chainParser), ValueSat: *satB1T2A3, }, { - addrDesc: addressToAddrDesc(addr2, d.chainParser), + AddrDesc: addressToAddrDesc(addr2, d.chainParser), ValueSat: *satB1T1A2, }, }, Outputs: []TxOutput{ { - addrDesc: addressToAddrDesc(addr6, d.chainParser), + AddrDesc: addressToAddrDesc(addr6, d.chainParser), Spent: true, ValueSat: *satB2T1A6, }, { - addrDesc: addressToAddrDesc(addr7, d.chainParser), + AddrDesc: addressToAddrDesc(addr7, d.chainParser), Spent: false, ValueSat: *satB2T1A7, }, @@ -1001,17 +1001,17 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 123, Inputs: []TxInput{ { - addrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + AddrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), ValueSat: *big.NewInt(0), }, { - addrDesc: addressToAddrDesc("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), + AddrDesc: addressToAddrDesc("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), ValueSat: *big.NewInt(1234123421342341234), }, }, Outputs: []TxOutput{ { - addrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + AddrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), ValueSat: *big.NewInt(1), Spent: true, }, @@ -1025,39 +1025,39 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 12345, Inputs: []TxInput{ { - addrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + AddrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), ValueSat: *big.NewInt(9011000000), }, { - addrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + AddrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), ValueSat: *big.NewInt(8011000000), }, { - addrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + AddrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), ValueSat: *big.NewInt(7011000000), }, }, Outputs: []TxOutput{ { - addrDesc: addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + AddrDesc: addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), ValueSat: *big.NewInt(5011000000), Spent: true, }, { - addrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + AddrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), ValueSat: *big.NewInt(6011000000), }, { - addrDesc: addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + AddrDesc: addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), ValueSat: *big.NewInt(7011000000), Spent: true, }, { - addrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + AddrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), ValueSat: *big.NewInt(999900000), }, { - addrDesc: addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + AddrDesc: addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), ValueSat: *big.NewInt(5000000000), Spent: true, }, @@ -1071,17 +1071,17 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 123456789, Inputs: []TxInput{ { - addrDesc: []byte{}, + AddrDesc: []byte{}, ValueSat: *big.NewInt(1234), }, }, Outputs: []TxOutput{ { - addrDesc: []byte{}, + AddrDesc: []byte{}, ValueSat: *big.NewInt(5678), }, { - addrDesc: []byte{}, + AddrDesc: []byte{}, ValueSat: *big.NewInt(98), Spent: true, }, From 6f257851093f21d2f8570312711341d800eb74fa Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 31 Aug 2018 14:03:06 +0200 Subject: [PATCH 047/123] Bcash address cashaddr/legacy handling --- bchain/coins/bch/bcashparser.go | 73 ++----- bchain/coins/bch/bcashparser_test.go | 275 ++++++++++++++----------- bchain/coins/btc/bitcoinparser_test.go | 2 +- 3 files changed, 174 insertions(+), 176 deletions(-) diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 2c804867..62f4c6b2 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -128,64 +128,21 @@ func (p *BCashParser) outputScriptToAddresses(script []byte) ([]string, bool, er if err != nil { return nil, false, err } + // EncodeAddress returns CashAddr address addr := a.EncodeAddress() + if p.AddressFormat == Legacy { + da, err := address.NewFromString(addr) + if err != nil { + return nil, false, err + } + ca, err := da.Legacy() + if err != nil { + return nil, false, err + } + addr, err = ca.Encode() + if err != nil { + return nil, false, err + } + } return []string{addr}, len(addr) > 0, nil } - -type bcashAddress struct { - addr string -} - -func newBCashAddress(addr string, format AddressFormat) (*bcashAddress, error) { - if isCashAddr(addr) && format == CashAddr { - return &bcashAddress{addr: addr}, nil - } - - da, err := address.NewFromString(addr) - if err != nil { - return nil, err - } - var ea string - switch format { - case CashAddr: - if a, err := da.CashAddress(); err != nil { - return nil, err - } else { - ea, err = a.Encode() - if err != nil { - return nil, err - } - } - - case Legacy: - if a, err := da.Legacy(); err != nil { - return nil, err - } else { - ea, err = a.Encode() - if err != nil { - return nil, err - } - } - default: - return nil, fmt.Errorf("Unknown address format: %d", format) - } - return &bcashAddress{addr: ea}, nil -} - -func (a *bcashAddress) String() string { - return a.addr -} - -func (a *bcashAddress) AreEqual(addr string) bool { - return a.String() == addr -} - -func (a *bcashAddress) InSlice(addrs []string) bool { - ea := a.String() - for _, addr := range addrs { - if ea == addr { - return true - } - } - return false -} diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index ef2867fe..4b3a0e68 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -5,127 +5,177 @@ package bch import ( "blockbook/bchain" "blockbook/bchain/coins/btc" - "bytes" "encoding/hex" "math/big" "reflect" "testing" ) -func TestBcashAddressEncodeAddress(t *testing.T) { - addr1, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", Legacy) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got1 := addr1.String() - if got1 != "13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji" { - t.Errorf("String() got1 = %v, want %v", got1, "13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji") - } - addr2, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", CashAddr) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got2 := addr2.String() - if got2 != "bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf" { - t.Errorf("String() got2 = %v, want %v", got2, "bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf") - } -} - -func TestBcashAddressAreEqual(t *testing.T) { - addr1, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", Legacy) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - addr2, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", CashAddr) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got1 := addr1.AreEqual("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji") - if got1 != true { - t.Errorf("AreEqual() got1 = %v, want %v", got1, true) - } - got2 := addr2.AreEqual("bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf") - if got2 != true { - t.Errorf("AreEqual() got2 = %v, want %v", got2, true) - } - got3 := addr1.AreEqual("1HoKgKQh7ZNomWURmS9Tk3z8JM2MWm7S1w") - if got3 != false { - t.Errorf("AreEqual() got3 = %v, want %v", got3, false) - } - got4 := addr2.AreEqual("bitcoincash:qzuyf0gpqj7q5wfck3nyghhklju7r0k3ksmq6d0vch") - if got4 != false { - t.Errorf("AreEqual() got4 = %v, want %v", got4, false) - } -} - -func TestBcashAddressInSlice(t *testing.T) { - addr1, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", Legacy) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - addr2, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", CashAddr) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got1 := addr1.InSlice([]string{"13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", "1HoKgKQh7ZNomWURmS9Tk3z8JM2MWm7S1w"}) - if got1 != true { - t.Errorf("InSlice() got1 = %v, want %v", got1, true) - } - got2 := addr2.InSlice([]string{"bitcoincash:qzuyf0gpqj7q5wfck3nyghhklju7r0k3ksmq6d0vch", "bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf"}) - if got2 != true { - t.Errorf("InSlice() got2 = %v, want %v", got2, true) - } - got3 := addr1.InSlice([]string{"1HoKgKQh7ZNomWURmS9Tk3z8JM2MWm7S1w", "1E6Np6dUPYpBSdLMLuwBF8sRQ3cngdaRRY"}) - if got3 != false { - t.Errorf("InSlice() got3 = %v, want %v", got3, false) - } - got4 := addr2.InSlice([]string{"bitcoincash:qzuyf0gpqj7q5wfck3nyghhklju7r0k3ksmq6d0vch", "bitcoincash:qz8emmpenqgeg7et8xsz8prvhy6cqcalyyjcamt7e9"}) - if got4 != false { - t.Errorf("InSlice() got4 = %v, want %v", got4, false) - } -} - func Test_GetAddrDescFromAddress(t *testing.T) { - parser, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) - if err != nil { - t.Errorf("NewBCashParser() error = %v", err) - return + mainParserCashAddr, mainParserLegacy, testParserCashAddr, _ := setupParsers(t) + tests := []struct { + name string + parser *BCashParser + addresses []string + hex string + wantErr bool + }{ + { + name: "test-P2PKH-0", + parser: testParserCashAddr, + addresses: []string{"mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr"}, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "test-P2PKH-1", + parser: testParserCashAddr, + addresses: []string{"bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm"}, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserLegacy, + addresses: []string{"129HiRqekqPVucKy2M8zsqvafGgKypciPp"}, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqxgjelx8qk85t9xfk8g2zlunxmhxms6p55xarv2r5"}, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2SH-0", + parser: mainParserCashAddr, + addresses: []string{"3EBEFWPtDYWCNszQ7etoqtWmmygccayLiH"}, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, + { + name: "main-P2SH-1", + parser: mainParserLegacy, + addresses: []string{"bitcoincash:pzy0wuj9pjps5v8dmlwq32fatu4wrgcwzuayq5nfhh"}, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, } - want, err := hex.DecodeString("76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac") - if err != nil { - panic(err) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.parser.GetAddrDescFromAddress(tt.addresses[0]) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.hex) { + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.hex) + } + }) } - got1, err := parser.GetAddrDescFromAddress("mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr") - if err != nil { - t.Errorf("GetAddrDescFromAddress() error = %v", err) - return +} + +func Test_GetAddressesFromAddrDesc(t *testing.T) { + mainParserCashAddr, mainParserLegacy, testParserCashAddr, testParserLegacy := setupParsers(t) + tests := []struct { + name string + parser *BCashParser + addresses []string + hex string + wantErr bool + }{ + { + name: "test-P2PKH-0", + parser: testParserLegacy, + addresses: []string{"mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr"}, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "test-P2PKH-1", + parser: testParserCashAddr, + addresses: []string{"bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm"}, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserLegacy, + addresses: []string{"129HiRqekqPVucKy2M8zsqvafGgKypciPp"}, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqxgjelx8qk85t9xfk8g2zlunxmhxms6p55xarv2r5"}, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2SH-0", + parser: mainParserLegacy, + addresses: []string{"3EBEFWPtDYWCNszQ7etoqtWmmygccayLiH"}, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, + { + name: "main-P2SH-1", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:pzy0wuj9pjps5v8dmlwq32fatu4wrgcwzuayq5nfhh"}, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, } - if !bytes.Equal(got1, want) { - t.Errorf("GetAddrDescFromAddress() got1 = %v, want %v", got1, want) - } - got2, err := parser.GetAddrDescFromAddress("bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm") - if err != nil { - t.Errorf("GetAddrDescFromAddress() error = %v", err) - return - } - if !bytes.Equal(got2, want) { - t.Errorf("GetAddrDescFromAddress() got2 = %v, want %v", got2, want) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.hex) + got, got2, err := tt.parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.addresses) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.addresses) + } + if !reflect.DeepEqual(got2, true) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, true) + } + }) } } var ( testTx1, testTx2 bchain.Tx - - testTxPacked1 = "0001e2408ba8d7af5401000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700" - testTxPacked2 = "0007c91a899ab7da6a010000000001019d64f0c72a0d206001decbffaa722eb1044534c74eee7a5df8318e42a4323ec10000000017160014550da1f5d25a9dae2eafd6902b4194c4c6500af6ffffffff02809698000000000017a914cd668d781ece600efa4b2404dc91fd26b8b8aed8870553d7360000000017a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a8702473044022076aba4ad559616905fa51d4ddd357fc1fdb428d40cb388e042cdd1da4a1b7357022011916f90c712ead9a66d5f058252efd280439ad8956a967e95d437d246710bc9012102a80a5964c5612bb769ef73147b2cf3c149bc0fd4ecb02f8097629c94ab013ffd00000000" + testTxPacked1 = "0001e2408ba8d7af5401000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700" + testTxPacked2 = "0007c91a899ab7da6a010000000001019d64f0c72a0d206001decbffaa722eb1044534c74eee7a5df8318e42a4323ec10000000017160014550da1f5d25a9dae2eafd6902b4194c4c6500af6ffffffff02809698000000000017a914cd668d781ece600efa4b2404dc91fd26b8b8aed8870553d7360000000017a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a8702473044022076aba4ad559616905fa51d4ddd357fc1fdb428d40cb388e042cdd1da4a1b7357022011916f90c712ead9a66d5f058252efd280439ad8956a967e95d437d246710bc9012102a80a5964c5612bb769ef73147b2cf3c149bc0fd4ecb02f8097629c94ab013ffd00000000" ) +func setupParsers(t *testing.T) (mainParserCashAddr, mainParserLegacy, testParserCashAddr, testParserLegacy *BCashParser) { + parser1, err := NewBCashParser(GetChainParams("main"), &btc.Configuration{AddressFormat: "cashaddr"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + parser2, err := NewBCashParser(GetChainParams("main"), &btc.Configuration{AddressFormat: "legacy"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + parser3, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "cashaddr"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + parser4, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + return parser1, parser2, parser3, parser4 +} + func init() { testTx1 = bchain.Tx{ @@ -200,16 +250,7 @@ func init() { } func Test_UnpackTx(t *testing.T) { - parser1, err := NewBCashParser(GetChainParams("main"), &btc.Configuration{AddressFormat: "legacy"}) - if err != nil { - t.Errorf("NewBCashParser() error = %v", err) - return - } - parser2, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) - if err != nil { - t.Errorf("NewBCashParser() error = %v", err) - return - } + mainParser, _, testParser, _ := setupParsers(t) type args struct { packedTx string @@ -223,10 +264,10 @@ func Test_UnpackTx(t *testing.T) { wantErr bool }{ { - name: "btc-1", + name: "bcash-1", args: args{ packedTx: testTxPacked1, - parser: parser1, + parser: mainParser, }, want: &testTx1, want1: 123456, @@ -236,7 +277,7 @@ func Test_UnpackTx(t *testing.T) { name: "testnet-1", args: args{ packedTx: testTxPacked2, - parser: parser2, + parser: testParser, }, want: &testTx2, want1: 510234, diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 74f7b4cb..2350f3ef 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -124,7 +124,7 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { b, _ := hex.DecodeString(tt.args.script) got, got2, err := parser.GetAddressesFromAddrDesc(b) if (err != nil) != tt.wantErr { - t.Errorf("outputScriptToAddresses() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { From 1109cd787823389bc325074e5382b14cefaae7bc Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 31 Aug 2018 15:23:04 +0200 Subject: [PATCH 048/123] Use AddressDescriptor for queries in api worker --- api/types.go | 35 ++++++++++--------- api/worker.go | 82 +++++++++++++++++++++++++++------------------ static/css/main.css | 6 +++- 3 files changed, 75 insertions(+), 48 deletions(-) diff --git a/api/types.go b/api/types.go index 5e465ecf..f882e647 100644 --- a/api/types.go +++ b/api/types.go @@ -1,6 +1,9 @@ package api -import "math/big" +import ( + "blockbook/bchain" + "math/big" +) type ApiError struct { Text string @@ -24,23 +27,25 @@ type ScriptSig struct { } type Vin struct { - Txid string `json:"txid"` - Vout uint32 `json:"vout"` - Sequence int64 `json:"sequence,omitempty"` - N int `json:"n"` - ScriptSig ScriptSig `json:"scriptSig"` - Addresses []string `json:"addresses"` - Searchable bool `json:"-"` - Value string `json:"value"` - ValueSat big.Int `json:"-"` + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Sequence int64 `json:"sequence,omitempty"` + N int `json:"n"` + ScriptSig ScriptSig `json:"scriptSig"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` } type ScriptPubKey struct { - Hex string `json:"hex"` - Asm string `json:"asm,omitempty"` - Addresses []string `json:"addresses"` - Searchable bool `json:"-"` - Type string `json:"type,omitempty"` + Hex string `json:"hex"` + Asm string `json:"asm,omitempty"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Type string `json:"type,omitempty"` } type Vout struct { Value string `json:"value"` diff --git a/api/worker.go b/api/worker.go index 05d0c55a..e22178b3 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "bytes" "fmt" "math/big" "time" @@ -33,12 +34,13 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is return w, nil } -func (w *Worker) getAddressesFromVout(vout *bchain.Vout) ([]string, bool, error) { +func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescriptor, []string, bool, error) { addrDesc, err := w.chainParser.GetAddrDescFromVout(vout) if err != nil { - return nil, false, err + return nil, nil, false, err } - return w.chainParser.GetAddressesFromAddrDesc(addrDesc) + a, s, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + return addrDesc, a, s, err } // GetTransaction reads transaction data from txid @@ -47,6 +49,10 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool if err != nil { return nil, NewApiError(fmt.Sprintf("Tx not found, %v", err), true) } + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } var blockhash string if bchainTx.Confirmations > 0 { blockhash, err = w.db.GetBlockHash(height) @@ -66,11 +72,11 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool // bchainVin.Txid=="" is coinbase transaction if bchainVin.Txid != "" { // load spending addresses from TxAddresses - ta, err := w.db.GetTxAddresses(bchainVin.Txid) + tas, err := w.db.GetTxAddresses(bchainVin.Txid) if err != nil { return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) } - if ta == nil { + if tas == nil { // mempool transactions are not in TxAddresses, all confirmed should be there, log a problem if bchainTx.Confirmations > 0 { glog.Warning("DB inconsistency: tx ", bchainVin.Txid, ": not found in txAddresses") @@ -83,14 +89,14 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] vin.ValueSat = vout.ValueSat - vin.Addresses, vin.Searchable, err = w.getAddressesFromVout(vout) + vin.AddrDesc, vin.Addresses, vin.Searchable, err = w.getAddressesFromVout(vout) if err != nil { glog.Errorf("getAddressesFromVout error %v, vout %+v", err, vout) } } } else { - if len(ta.Outputs) > int(vin.Vout) { - output := &ta.Outputs[vin.Vout] + if len(tas.Outputs) > int(vin.Vout) { + output := &tas.Outputs[vin.Vout] vin.ValueSat = output.ValueSat vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) vin.Addresses, vin.Searchable, err = output.Addresses(w.chainParser) @@ -112,9 +118,12 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex - vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = w.getAddressesFromVout(bchainVout) - if spendingTxs { - // TODO + vout.ScriptPubKey.AddrDesc, vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = w.getAddressesFromVout(bchainVout) + if ta != nil { + vout.Spent = ta.Outputs[i].Spent + if spendingTxs && vout.Spent { + // TODO + } } } // for coinbase transactions valIn is 0 @@ -143,11 +152,11 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool return r, nil } -func (w *Worker) getAddressTxids(address string, mempool bool) ([]string, error) { +func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool) ([]string, error) { var err error txids := make([]string, 0) if !mempool { - err = w.db.GetTransactions(address, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { + err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { txids = append(txids, txid) return nil }) @@ -155,7 +164,7 @@ func (w *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return nil, err } } else { - m, err := w.chain.GetMempoolTransactions(address) + m, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) if err != nil { return nil, err } @@ -164,25 +173,21 @@ func (w *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return txids, nil } -func (t *Tx) getAddrVoutValue(addr string) *big.Int { +func (t *Tx) getAddrVoutValue(addrDesc bchain.AddressDescriptor) *big.Int { var val big.Int for _, vout := range t.Vout { - for _, a := range vout.ScriptPubKey.Addresses { - if a == addr { - val.Add(&val, &vout.ValueSat) - } + if bytes.Equal(vout.ScriptPubKey.AddrDesc, addrDesc) { + val.Add(&val, &vout.ValueSat) } } return &val } -func (t *Tx) getAddrVinValue(addr string) *big.Int { +func (t *Tx) getAddrVinValue(addrDesc bchain.AddressDescriptor) *big.Int { var val big.Int for _, vin := range t.Vin { - for _, a := range vin.Addresses { - if a == addr { - val.Add(&val, &vin.ValueSat) - } + if bytes.Equal(vin.AddrDesc, addrDesc) { + val.Add(&val, &vin.ValueSat) } } return &val @@ -260,27 +265,37 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b if page < 0 { page = 0 } - ba, err := w.db.GetAddressBalance(address) + addrDesc, err := w.chainParser.GetAddrDescFromAddress(address) if err != nil { return nil, NewApiError(fmt.Sprintf("Address not found, %v", err), true) } - if ba == nil { - return nil, NewApiError("Address not found", true) + // ba can be nil if the address is only in mempool! + ba, err := w.db.GetAddrDescBalance(addrDesc) + if err != nil { + return nil, NewApiError(fmt.Sprintf("Address not found, %v", err), true) } - txc, err := w.getAddressTxids(address, false) + txc, err := w.getAddressTxids(addrDesc, false) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v false", address) } txc = UniqueTxidsInReverse(txc) var txm []string - // mempool only on the first page - if page == 0 { - txm, err = w.getAddressTxids(address, true) + // mempool only on the first page or if there are no confirmed transactions + if page == 0 || ba == nil { + if ba == nil { + ba = &db.AddrBalance{} + page = 0 + } + txm, err = w.getAddressTxids(addrDesc, true) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v true", address) } txm = UniqueTxidsInReverse(txm) } + // check if the address exist + if len(txc)+len(txm) == 0 { + return nil, NewApiError("Address not found", true) + } bestheight, _, err := w.db.GetBestBlock() if err != nil { return nil, errors.Annotatef(err, "GetBestBlock") @@ -309,7 +324,9 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b if err != nil { glog.Error("GetTransaction in mempool ", tx, ": ", err) } else { - uBalSat.Sub(tx.getAddrVoutValue(address), tx.getAddrVinValue(address)) + uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) + uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) + glog.Info("uBalSat ", uBalSat, tx.getAddrVoutValue(addrDesc), tx.getAddrVinValue(addrDesc)) txs[txi] = tx txi++ } @@ -347,6 +364,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), UnconfirmedTxApperances: len(txm), Transactions: txs[:txi], + Txids: nil, Page: page, TotalPages: totalPages, TxsOnPage: txsOnPage, diff --git a/static/css/main.css b/static/css/main.css index b52b682d..e2905211 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -177,6 +177,10 @@ h3 { padding: .4rem; } +.data-table span.ellipsis { + max-width: 100%; +} + .data { font-weight: bold; } @@ -188,4 +192,4 @@ h3 { .navbar-nav .nav-link { padding-right: 0; padding-left: .25rem; -} +} \ No newline at end of file From fdf1fe612f18878f976bc257ff9d0db5ef103ed0 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 2 Sep 2018 21:31:33 +0200 Subject: [PATCH 049/123] Stringer interface for AddressDescriptor --- bchain/types.go | 5 +++++ db/rocksdb.go | 11 ++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bchain/types.go b/bchain/types.go index e27bca57..00630f13 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -2,6 +2,7 @@ package bchain import ( "context" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -114,6 +115,10 @@ func (e *RPCError) Error() string { // AddressDescriptor is an opaque type obtained by parser.GetAddrDesc* methods type AddressDescriptor []byte +func (ad AddressDescriptor) String() string { + return "ad:" + hex.EncodeToString(ad) +} + // BlockChain defines common interface to block chain daemon type BlockChain interface { // life-cycle methods diff --git a/db/rocksdb.go b/db/rocksdb.go index 00967f0d..b16647cd 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -339,11 +339,10 @@ type blockTxs struct { func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc bchain.AddressDescriptor, logText string) { ad, _, err := d.chainParser.GetAddressesFromAddrDesc(addrDesc) - had := hex.EncodeToString(addrDesc) if err != nil { - glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", had, logText, valueSat.String(), err) + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", addrDesc, logText, valueSat.String(), err) } else { - glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, had, logText, valueSat.String()) + glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, addrDesc, logText, valueSat.String()) } valueSat.SetInt64(0) } @@ -1073,8 +1072,7 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, b.BalanceSat.Add(&b.BalanceSat, &t.ValueSat) } else { ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) - had := hex.EncodeToString(t.AddrDesc) - glog.Warningf("Balance for address %s (%s) not found", ad, had) + glog.Warningf("Balance for address %s (%s) not found", ad, t.AddrDesc) } s = string(inputs[i].btxID) sa, exist := txAddressesToUpdate[s] @@ -1110,8 +1108,7 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } } else { ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) - had := hex.EncodeToString(t.AddrDesc) - glog.Warningf("Balance for address %s (%s) not found", ad, had) + glog.Warningf("Balance for address %s (%s) not found", ad, t.AddrDesc) } } } From fdb069ef311005c99c01822ad84723ce67ac65d1 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 2 Sep 2018 21:32:20 +0200 Subject: [PATCH 050/123] Normalize address in worker.GetAddress --- api/worker.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/api/worker.go b/api/worker.go index e22178b3..695f9747 100644 --- a/api/worker.go +++ b/api/worker.go @@ -77,7 +77,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) } if tas == nil { - // mempool transactions are not in TxAddresses, all confirmed should be there, log a problem + // mempool transactions are not in TxAddresses but confirmed should be there, log a problem if bchainTx.Confirmations > 0 { glog.Warning("DB inconsistency: tx ", bchainVin.Txid, ": not found in txAddresses") } @@ -99,6 +99,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool output := &tas.Outputs[vin.Vout] vin.ValueSat = output.ValueSat vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + vin.AddrDesc = output.AddrDesc vin.Addresses, vin.Searchable, err = output.Addresses(w.chainParser) if err != nil { glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) @@ -274,6 +275,11 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b if err != nil { return nil, NewApiError(fmt.Sprintf("Address not found, %v", err), true) } + // convert the address to the format defined by the parser + addresses, _, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + if len(addresses) == 1 { + address = addresses[0] + } txc, err := w.getAddressTxids(addrDesc, false) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v false", address) @@ -326,7 +332,6 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } else { uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) - glog.Info("uBalSat ", uBalSat, tx.getAddrVoutValue(addrDesc), tx.getAddrVinValue(addrDesc)) txs[txi] = tx txi++ } From ed1bdb30f77d36be118f125dd53c555f5abab52a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 2 Sep 2018 21:32:32 +0200 Subject: [PATCH 051/123] Switch bcash to cashaddr format --- configs/coins/bcash.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index e1bfe518..78c74336 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -49,7 +49,7 @@ "block_chain": { "parse": true, "subversion": "/Bitcoin ABC:0.17.1/", - "address_format": "legacy", + "address_format": "cashaddr", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, From fe4b3668754fb6bb495564dee3f3e87d0f31628a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 3 Sep 2018 13:25:50 +0200 Subject: [PATCH 052/123] Unite scripts for port checking and registry of ports to one --- CONTRIBUTING.md | 5 +- ...go => check-and-generate-port-registry.go} | 66 +++++++++++++++ contrib/scripts/check-ports.go | 83 ------------------- docs/ports.md | 2 +- 4 files changed, 69 insertions(+), 87 deletions(-) rename contrib/scripts/{generate-port-registry.go => check-and-generate-port-registry.go} (80%) delete mode 100755 contrib/scripts/check-ports.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ffb9789..fefff9a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,9 +81,8 @@ In section *blockbook* update information how to build and configure Blockbook s Update *package_maintainer* and *package_maintainer_email* options in section *meta*. -Execute script *contrib/scripts/check-ports.go* that will check mandatory ports and uniquity of registered ports. - -Execute script *contrib/scripts/generate-port-registry.go* that will update *docs/ports.md*. +Execute script *go run contrib/scripts/check-and-generate-port-registry.go -w* that checks mandatory ports and +uniqueness of ports and updates registry of ports *docs/ports.md*. Now you can try generate package definitions as described above in order to check outputs. diff --git a/contrib/scripts/generate-port-registry.go b/contrib/scripts/check-and-generate-port-registry.go similarity index 80% rename from contrib/scripts/generate-port-registry.go rename to contrib/scripts/check-and-generate-port-registry.go index 20375eea..94e7c2c4 100755 --- a/contrib/scripts/generate-port-registry.go +++ b/contrib/scripts/check-and-generate-port-registry.go @@ -36,6 +36,67 @@ type Config struct { Ports map[string]uint16 `json:"ports"` } +func checkPorts() int { + ports := make(map[uint16][]string) + status := 0 + + files, err := ioutil.ReadDir(inputDir) + if err != nil { + panic(err) + } + + for _, fi := range files { + if fi.IsDir() || fi.Name()[0] == '.' { + continue + } + + path := filepath.Join(inputDir, fi.Name()) + f, err := os.Open(path) + if err != nil { + panic(fmt.Errorf("%s: %s", path, err)) + } + defer f.Close() + + v := Config{} + d := json.NewDecoder(f) + err = d.Decode(&v) + if err != nil { + panic(fmt.Errorf("%s: json: %s", path, err)) + } + + if _, ok := v.Ports["blockbook_internal"]; !ok { + fmt.Printf("%s: missing blockbook_internal port\n", v.Coin.Name) + status = 1 + } + if _, ok := v.Ports["blockbook_public"]; !ok { + fmt.Printf("%s: missing blockbook_public port\n", v.Coin.Name) + status = 1 + } + if _, ok := v.Ports["backend_rpc"]; !ok { + fmt.Printf("%s: missing backend_rpc port\n", v.Coin.Name) + status = 1 + } + + for _, port := range v.Ports { + if port > 0 { + ports[port] = append(ports[port], v.Coin.Name) + } + } + } + + for port, coins := range ports { + if len(coins) > 1 { + fmt.Printf("port %d: registered by %q\n", port, coins) + status = 1 + } + } + + if status != 0 { + fmt.Println("Got some errors") + } + return status +} + func main() { output := "stdout" if len(os.Args) > 1 { @@ -48,6 +109,11 @@ func main() { } } + status := checkPorts() + if status != 0 { + os.Exit(status) + } + slice, err := loadPortInfo(inputDir) if err != nil { panic(err) diff --git a/contrib/scripts/check-ports.go b/contrib/scripts/check-ports.go deleted file mode 100755 index 1e509de1..00000000 --- a/contrib/scripts/check-ports.go +++ /dev/null @@ -1,83 +0,0 @@ -//usr/bin/go run $0 $@ ; exit -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" -) - -const configDir = "configs/coins" - -type Config struct { - Coin struct { - Name string `json:"name"` - } - Ports map[string]uint16 `json:"ports"` -} - -func main() { - ports := make(map[uint16][]string) - status := 0 - - files, err := ioutil.ReadDir(configDir) - if err != nil { - panic(err) - } - - for _, fi := range files { - if fi.IsDir() || fi.Name()[0] == '.' { - continue - } - - path := filepath.Join(configDir, fi.Name()) - f, err := os.Open(path) - if err != nil { - panic(fmt.Errorf("%s: %s", path, err)) - } - defer f.Close() - - v := Config{} - d := json.NewDecoder(f) - err = d.Decode(&v) - if err != nil { - panic(fmt.Errorf("%s: json: %s", path, err)) - } - - if _, ok := v.Ports["blockbook_internal"]; !ok { - fmt.Printf("%s: missing blockbook_internal port\n", v.Coin.Name) - status = 1 - } - if _, ok := v.Ports["blockbook_public"]; !ok { - fmt.Printf("%s: missing blockbook_public port\n", v.Coin.Name) - status = 1 - } - if _, ok := v.Ports["backend_rpc"]; !ok { - fmt.Printf("%s: missing backend_rpc port\n", v.Coin.Name) - status = 1 - } - - for _, port := range v.Ports { - if port > 0 { - ports[port] = append(ports[port], v.Coin.Name) - } - } - } - - for port, coins := range ports { - if len(coins) > 1 { - fmt.Printf("port %d: registered by %q\n", port, coins) - status = 1 - } - } - - if status == 0 { - fmt.Println("OK") - } else { - fmt.Println("Got some errors") - } - - os.Exit(status) -} diff --git a/docs/ports.md b/docs/ports.md index 77072825..e400f01d 100644 --- a/docs/ports.md +++ b/docs/ports.md @@ -8,7 +8,7 @@ | Dash | 9033 | 9133 | 8033 | 38333 | | Litecoin | 9034 | 9134 | 8034 | 38334 | | Bgold | 9035 | 9135 | 8035 | 38335 | -| Ethereum | 9036 | 9136 | 8036 | 38336 p2p, 8136 http | +| Ethereum | 9036 | 9136 | 8036 | 8136 http, 38336 p2p | | Ethereum Classic | 9037 | 9137 | 8037 | | | Dogecoin | 9038 | 9138 | 8038 | 38338 | | Namecoin | 9039 | 9139 | 8039 | 38339 | From 1b69a62ab8818a35fcc0743cebd98b49f3235a9d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 3 Sep 2018 14:10:28 +0200 Subject: [PATCH 053/123] Process OP_RETURN script in Bcash --- api/worker.go | 4 +- bchain/coins/bch/bcashparser.go | 9 +++ bchain/coins/bch/bcashparser_test.go | 106 +++++++++++++++++---------- bchain/coins/btc/bitcoinparser.go | 49 ++++++++----- 4 files changed, 109 insertions(+), 59 deletions(-) diff --git a/api/worker.go b/api/worker.go index 695f9747..05a9aca9 100644 --- a/api/worker.go +++ b/api/worker.go @@ -223,7 +223,7 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn valInSat.Add(&valInSat, &vin.ValueSat) vin.Addresses, vin.Searchable, err = tai.Addresses(w.chainParser) if err != nil { - glog.Errorf("tai.Addresses error %v, tx %v, input %v", err, txid, i) + glog.Errorf("tai.Addresses error %v, tx %v, input %v, tai %+v", err, txid, i, tai) } } vouts := make([]Vout, len(ta.Outputs)) @@ -236,7 +236,7 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn valOutSat.Add(&valOutSat, &vout.ValueSat) vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = tao.Addresses(w.chainParser) if err != nil { - glog.Errorf("tai.Addresses error %v, tx %v, output %v", err, txid, i) + glog.Errorf("tai.Addresses error %v, tx %v, output %v, tao %+v", err, txid, i, tao) } } // for coinbase transactions valIn is 0 diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 62f4c6b2..428e80b4 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -126,6 +126,15 @@ func isCashAddr(addr string) bool { func (p *BCashParser) outputScriptToAddresses(script []byte) ([]string, bool, error) { a, err := bchutil.ExtractPkScriptAddrs(script, p.Params) if err != nil { + // do not return unknown script type error as error + if err.Error() == "unknown script type" { + // try OP_RETURN script + or := btc.TryParseOPReturn(script) + if or != "" { + return []string{or}, false, nil + } + return []string{}, false, nil + } return nil, false, err } // EncodeAddress returns CashAddr address diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index 4b3a0e68..7d98cea6 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -82,53 +82,85 @@ func Test_GetAddrDescFromAddress(t *testing.T) { func Test_GetAddressesFromAddrDesc(t *testing.T) { mainParserCashAddr, mainParserLegacy, testParserCashAddr, testParserLegacy := setupParsers(t) tests := []struct { - name string - parser *BCashParser - addresses []string - hex string - wantErr bool + name string + parser *BCashParser + addresses []string + searchable bool + hex string + wantErr bool }{ { - name: "test-P2PKH-0", - parser: testParserLegacy, - addresses: []string{"mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr"}, - hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", - wantErr: false, + name: "test-P2PKH-0", + parser: testParserLegacy, + addresses: []string{"mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr"}, + searchable: true, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + + wantErr: false, }, { - name: "test-P2PKH-1", - parser: testParserCashAddr, - addresses: []string{"bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm"}, - hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", - wantErr: false, + name: "test-P2PKH-1", + parser: testParserCashAddr, + addresses: []string{"bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm"}, + searchable: true, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, }, { - name: "main-P2PKH-0", - parser: mainParserLegacy, - addresses: []string{"129HiRqekqPVucKy2M8zsqvafGgKypciPp"}, - hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", - wantErr: false, + name: "main-P2PKH-0", + parser: mainParserLegacy, + addresses: []string{"129HiRqekqPVucKy2M8zsqvafGgKypciPp"}, + searchable: true, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, }, { - name: "main-P2PKH-0", - parser: mainParserCashAddr, - addresses: []string{"bitcoincash:qqxgjelx8qk85t9xfk8g2zlunxmhxms6p55xarv2r5"}, - hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", - wantErr: false, + name: "main-P2PKH-0", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqxgjelx8qk85t9xfk8g2zlunxmhxms6p55xarv2r5"}, + searchable: true, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, }, { - name: "main-P2SH-0", - parser: mainParserLegacy, - addresses: []string{"3EBEFWPtDYWCNszQ7etoqtWmmygccayLiH"}, - hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", - wantErr: false, + name: "main-P2SH-0", + parser: mainParserLegacy, + addresses: []string{"3EBEFWPtDYWCNszQ7etoqtWmmygccayLiH"}, + searchable: true, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, }, { - name: "main-P2SH-1", - parser: mainParserCashAddr, - addresses: []string{"bitcoincash:pzy0wuj9pjps5v8dmlwq32fatu4wrgcwzuayq5nfhh"}, - hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", - wantErr: false, + name: "main-P2SH-1", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:pzy0wuj9pjps5v8dmlwq32fatu4wrgcwzuayq5nfhh"}, + searchable: true, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, + { + name: "OP_RETURN ascii", + parser: mainParserCashAddr, + addresses: []string{"OP_RETURN (ahoj)"}, + searchable: false, + hex: "6a0461686f6a", + wantErr: false, + }, + { + name: "OP_RETURN hex", + parser: mainParserCashAddr, + addresses: []string{"OP_RETURN 07 2020f1686f6a20"}, + searchable: false, + hex: "6a072020f1686f6a20", + wantErr: false, + }, + { + name: "empty", + parser: mainParserCashAddr, + addresses: []string{}, + searchable: false, + hex: "", + wantErr: false, }, } @@ -143,8 +175,8 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { if !reflect.DeepEqual(got, tt.addresses) { t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.addresses) } - if !reflect.DeepEqual(got2, true) { - t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, true) + if !reflect.DeepEqual(got2, tt.searchable) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.searchable) } }) } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index a4745be0..d995a735 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -84,6 +84,31 @@ func (p *BitcoinParser) addressToOutputScript(address string) ([]byte, error) { return script, nil } +// TryParseOPReturn tries to process OP_RETURN script and return its string representation +func TryParseOPReturn(script []byte) string { + if len(script) > 1 && script[0] == txscript.OP_RETURN { + l := int(script[1]) + data := script[2:] + if l == len(data) { + isASCII := true + for _, c := range data { + if c < 32 || c > 127 { + isASCII = false + break + } + } + var ed string + if isASCII { + ed = "(" + string(data) + ")" + } else { + ed = hex.EncodeToString([]byte{byte(l)}) + " " + hex.EncodeToString(data) + } + return "OP_RETURN " + ed + } + } + return "" +} + // outputScriptToAddresses converts ScriptPubKey to bitcoin addresses func (p *BitcoinParser) outputScriptToAddresses(script []byte) ([]string, bool, error) { sc, addresses, _, err := txscript.ExtractPkScriptAddrs(script, p.Params) @@ -97,26 +122,10 @@ func (p *BitcoinParser) outputScriptToAddresses(script []byte) ([]string, bool, var s bool if sc != txscript.NonStandardTy && sc != txscript.NullDataTy { s = true - } else { - if len(script) > 1 && script[0] == txscript.OP_RETURN && len(rv) == 0 { - l := int(script[1]) - data := script[2:] - if l == len(data) { - isASCII := true - for _, c := range data { - if c < 32 || c > 127 { - isASCII = false - break - } - } - var ed string - if isASCII { - ed = "(" + string(data) + ")" - } else { - ed = hex.EncodeToString([]byte{byte(l)}) + " " + hex.EncodeToString(data) - } - rv = []string{"OP_RETURN " + ed} - } + } else if len(rv) == 0 { + or := TryParseOPReturn(script) + if or != "" { + rv = []string{or} } } return rv, s, nil From 984f24a76dece5f8e0dbe3e39c605db76548784e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 3 Sep 2018 15:44:03 +0200 Subject: [PATCH 054/123] Handle pay-to-pubkey addresses in Bcash --- bchain/coins/bch/bcashparser.go | 26 ++++++++++++++++++++------ bchain/coins/bch/bcashparser_test.go | 8 ++++++++ static/templates/txdetail.html | 9 ++++++++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 428e80b4..3aae771b 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -128,14 +128,28 @@ func (p *BCashParser) outputScriptToAddresses(script []byte) ([]string, bool, er if err != nil { // do not return unknown script type error as error if err.Error() == "unknown script type" { - // try OP_RETURN script - or := btc.TryParseOPReturn(script) - if or != "" { - return []string{or}, false, nil + // try bitcoin parser to parse P2PK scripts - kind of hack as bchutil does not support pay-to-pubkey + _, addresses, _, err := txscript.ExtractPkScriptAddrs(script, p.Params) + if err == nil && len(addresses) == 1 { + // convert the address back to output script and back to get from bitcoin to bcash + s, err := p.addressToOutputScript(addresses[0].EncodeAddress()) + if err == nil { + a, err = bchutil.ExtractPkScriptAddrs(s, p.Params) + if err != nil { + return []string{}, false, nil + } + } + } else { + // try OP_RETURN script + or := btc.TryParseOPReturn(script) + if or != "" { + return []string{or}, false, nil + } + return []string{}, false, nil } - return []string{}, false, nil + } else { + return nil, false, err } - return nil, false, err } // EncodeAddress returns CashAddr address addr := a.EncodeAddress() diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index 7d98cea6..d9e38ac7 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -138,6 +138,14 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", wantErr: false, }, + { + name: "main-P2PK", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqr95pwp0w5jqnh9vcjl4qm4x45atr0er57n49pq75"}, + searchable: true, + hex: "2103db3c3977c5165058bf38c46f72d32f4e872112dbafc13083a948676165cd1603ac", + wantErr: false, + }, { name: "OP_RETURN ascii", parser: mainParserCashAddr, diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index f352700c..ac22828b 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -51,7 +51,14 @@ {{else}} Unparsed address {{end}} - {{formatAmount $vout.Value}} {{$cs}} + + {{formatAmount $vout.Value}} {{$cs}} + {{if $vout.Spent}} + (S) + {{else}} + (U) + {{end}} + {{end}} From 63ad3ffff2e23e629f2375950111f9f9b444159b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 3 Sep 2018 17:25:57 +0200 Subject: [PATCH 055/123] Link spending transaction to output in explorer --- api/worker.go | 39 +++++++++++++++++++++++++++-- static/templates/txdetail.html | 45 ++++++++++++++++------------------ 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/api/worker.go b/api/worker.go index 05a9aca9..7d451dd1 100644 --- a/api/worker.go +++ b/api/worker.go @@ -45,6 +45,7 @@ func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescript // GetTransaction reads transaction data from txid func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool) (*Tx, error) { + start := time.Now() bchainTx, height, err := w.txCache.GetTransaction(txid, bestheight) if err != nil { return nil, NewApiError(fmt.Sprintf("Tx not found, %v", err), true) @@ -123,7 +124,39 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool if ta != nil { vout.Spent = ta.Outputs[i].Spent if spendingTxs && vout.Spent { - // TODO + // find transaction that spent this output + // there is not an index, it must be found in addresses -> txaddresses -> tx + // given that each step is more and more selective, it is not + err = w.db.GetAddrDescTransactions(vout.ScriptPubKey.AddrDesc, height, ^uint32(0), func(t string, index uint32, isOutput bool) error { + if isOutput == false { + tsp, err := w.db.GetTxAddresses(t) + if err != nil { + glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses") + } else { + if len(tsp.Inputs) > int(index) { + if tsp.Inputs[index].ValueSat.Cmp(&vout.ValueSat) == 0 { + spentTx, spentHeight, err := w.txCache.GetTransaction(t, bestheight) + if err != nil { + glog.Warning("Tx ", t, ": not found") + } else { + if len(spentTx.Vin) > int(index) { + if spentTx.Vin[index].Txid == bchainTx.Txid { + vout.SpentTxID = t + vout.SpentHeight = int(spentHeight) + vout.SpentIndex = int(index) + return &db.StopIteration{} + } + } + } + } + } + } + } + return nil + }) + if err != nil { + glog.Errorf("GetAddrDescTransactions error %v, %v, output %v", err, vout.ScriptPubKey.AddrDesc) + } } } } @@ -150,6 +183,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool Vin: vins, Vout: vouts, } + glog.Info("GetTransaction ", txid, " finished in ", time.Since(start)) return r, nil } @@ -238,6 +272,7 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn if err != nil { glog.Errorf("tai.Addresses error %v, tx %v, output %v, tao %+v", err, txid, i, tao) } + vout.Spent = tao.Spent } // for coinbase transactions valIn is 0 feesSat.Sub(&valInSat, &valOutSat) @@ -374,6 +409,6 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b TotalPages: totalPages, TxsOnPage: txsOnPage, } - glog.Info(address, " finished in ", time.Since(start)) + glog.Info("GetAddress ", address, " finished in ", time.Since(start)) return r, nil } diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index ac22828b..9fafc7ec 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -4,30 +4,30 @@
{{$tx.Txid}}
- {{if $tx.Confirmations}} + {{- if $tx.Confirmations -}}
mined {{formatUnixTime $tx.Blocktime}}
- {{end}} + {{- end -}}
- {{range $vin := $tx.Vin}} + {{- range $vin := $tx.Vin -}} - {{end}} + {{- end -}}
- {{range $a := $vin.Addresses}} + {{- range $a := $vin.Addresses}} {{if and (ne $a $addr) $vin.Searchable}}{{$a}}{{else}}{{$a}}{{end}} - {{else}} + {{- else -}} No Inputs (Newly Generated Coins) - {{end}}{{if $vin.Addresses}} + {{- end -}}{{- if $vin.Addresses -}} {{formatAmount $vin.Value}} {{$cs}} - {{end}} + {{- end -}}
@@ -41,27 +41,24 @@
- {{range $vout := $tx.Vout}} + {{- range $vout := $tx.Vout -}} - {{end}} + {{- end -}}
- {{range $a := $vout.ScriptPubKey.Addresses}} + {{- range $a := $vout.ScriptPubKey.Addresses -}} - {{if and (ne $a $addr) $vout.ScriptPubKey.Searchable}}{{$a}}{{else}}{{$a}}{{end}} + {{- if and (ne $a $addr) $vout.ScriptPubKey.Searchable}}{{$a}}{{else}}{{$a}}{{- end -}} - {{else}} + {{- else -}} Unparsed address - {{end}} + {{- end -}} - {{formatAmount $vout.Value}} {{$cs}} - {{if $vout.Spent}} - (S) - {{else}} + {{formatAmount $vout.Value}} {{$cs}}{{if $vout.Spent}}{{if $vout.SpentTxID}}(S){{else}}(S){{end}}{{else -}} (U) - {{end}} + {{- end -}}
@@ -69,16 +66,16 @@
- {{if $tx.Fees}} + {{- if $tx.Fees -}} Fee: {{formatAmount $tx.Fees}} {{$cs}} - {{end}} + {{- end -}}
- {{if $tx.Confirmations}} + {{- if $tx.Confirmations -}} {{$tx.Confirmations}} Confirmations - {{else}} + {{- else -}} Unconfirmed Transaction! - {{end}} + {{- end -}} {{formatAmount $tx.ValueOut}} {{$cs}}
From 2033dba1691e182b4fd0b4c325664dc27e1e11bc Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 4 Sep 2018 14:34:15 +0200 Subject: [PATCH 056/123] Return address array of txids in api call GetAddress --- api/types.go | 4 +-- api/worker.go | 82 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/api/types.go b/api/types.go index f882e647..ed0a1a2d 100644 --- a/api/types.go +++ b/api/types.go @@ -84,8 +84,8 @@ type Address struct { UnconfirmedBalance string `json:"unconfirmedBalance"` UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"transactions,omitempty"` - Txids []string `json:"transactions,omitempty"` // this is intentional, we return either Transactions or Txids + Transactions []*Tx `json:"txs,omitempty"` + Txids []string `json:"transactions,omitempty"` Page int `json:"page"` TotalPages int `json:"totalPages"` TxsOnPage int `json:"txsOnPage"` diff --git a/api/worker.go b/api/worker.go index 7d451dd1..405e57a9 100644 --- a/api/worker.go +++ b/api/worker.go @@ -126,7 +126,6 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool if spendingTxs && vout.Spent { // find transaction that spent this output // there is not an index, it must be found in addresses -> txaddresses -> tx - // given that each step is more and more selective, it is not err = w.db.GetAddrDescTransactions(vout.ScriptPubKey.AddrDesc, height, ^uint32(0), func(t string, index uint32, isOutput bool) error { if isOutput == false { tsp, err := w.db.GetTxAddresses(t) @@ -321,18 +320,16 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } txc = UniqueTxidsInReverse(txc) var txm []string - // mempool only on the first page or if there are no confirmed transactions - if page == 0 || ba == nil { - if ba == nil { - ba = &db.AddrBalance{} - page = 0 - } - txm, err = w.getAddressTxids(addrDesc, true) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v true", address) - } - txm = UniqueTxidsInReverse(txm) + // if there are only unconfirmed transactions, ba is nil + if ba == nil { + ba = &db.AddrBalance{} + page = 0 } + txm, err = w.getAddressTxids(addrDesc, true) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v true", address) + } + txm = UniqueTxidsInReverse(txm) // check if the address exist if len(txc)+len(txm) == 0 { return nil, NewApiError("Address not found", true) @@ -355,7 +352,13 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b if to > len(txc) { to = len(txc) } - txs := make([]*Tx, len(txm)+to-from) + var txs []*Tx + var txids []string + if onlyTxids { + txids = make([]string, len(txm)+to-from) + } else { + txs = make([]*Tx, len(txm)+to-from) + } txi := 0 // load mempool transactions var uBalSat big.Int @@ -367,8 +370,14 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } else { uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) - txs[txi] = tx - txi++ + if page > 0 { + if onlyTxids { + txids[txi] = tx.Txid + } else { + txs[txi] = tx + } + txi++ + } } } if len(txc) != int(ba.Txs) { @@ -376,25 +385,32 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } for i := from; i < to; i++ { txid := txc[i] - ta, err := w.db.GetTxAddresses(txid) - if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + if onlyTxids { + txids[txi] = txid + } else { + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + bi, err := w.db.GetBlockInfo(ta.Height) + if err != nil { + return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) + } + if bi == nil { + glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) } - if ta == nil { - glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") - continue - } - bi, err := w.db.GetBlockInfo(ta.Height) - if err != nil { - return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) - } - if bi == nil { - glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") - continue - } - txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) txi++ } + if !onlyTxids { + txs = txs[:txi] + } r := &Address{ AddrStr: address, Balance: w.chainParser.AmountToDecimalString(&ba.BalanceSat), @@ -403,8 +419,8 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b TxApperances: len(txc), UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), UnconfirmedTxApperances: len(txm), - Transactions: txs[:txi], - Txids: nil, + Transactions: txs, + Txids: txids, Page: page, TotalPages: totalPages, TxsOnPage: txsOnPage, From fb0c766ee4c4f1f144ae3e6939c1f5dd1d34fb41 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 5 Sep 2018 15:13:45 +0200 Subject: [PATCH 057/123] Add paging to explorer --- api/worker.go | 20 +++++++--- server/public.go | 70 +++++++++++++++++++++++++++++++++-- static/css/main.css | 34 +++++++++++++++++ static/templates/address.html | 16 +++++--- static/templates/paging.html | 11 ++++++ 5 files changed, 135 insertions(+), 16 deletions(-) create mode 100644 static/templates/paging.html diff --git a/api/worker.go b/api/worker.go index 405e57a9..7320134d 100644 --- a/api/worker.go +++ b/api/worker.go @@ -182,7 +182,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool Vin: vins, Vout: vouts, } - glog.Info("GetTransaction ", txid, " finished in ", time.Since(start)) + if spendingTxs { + glog.Info("GetTransaction ", txid, " finished in ", time.Since(start)) + } return r, nil } @@ -297,6 +299,7 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn // GetAddress computes address value and gets transactions for given address func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids bool) (*Address, error) { start := time.Now() + page-- if page < 0 { page = 0 } @@ -340,7 +343,10 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } // paging from := page * txsOnPage - totalPages := len(txc) / txsOnPage + totalPages := (len(txc) - 1) / txsOnPage + if totalPages < 0 { + totalPages = 0 + } if from >= len(txc) { page = totalPages - 1 if page < 0 { @@ -370,7 +376,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } else { uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) - if page > 0 { + if page == 0 { if onlyTxids { txids[txi] = tx.Txid } else { @@ -408,7 +414,9 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b } txi++ } - if !onlyTxids { + if onlyTxids { + txids = txids[:txi] + } else { txs = txs[:txi] } r := &Address{ @@ -421,8 +429,8 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b UnconfirmedTxApperances: len(txm), Transactions: txs, Txids: txids, - Page: page, - TotalPages: totalPages, + Page: page + 1, + TotalPages: totalPages + 1, TxsOnPage: txsOnPage, } glog.Info("GetAddress ", address, " finished in ", time.Since(start)) diff --git a/server/public.go b/server/public.go index 9b9800b7..7224f61a 100644 --- a/server/public.go +++ b/server/public.go @@ -20,7 +20,7 @@ import ( ) const blockbookAbout = "Blockbook - blockchain indexer for TREZOR wallet https://trezor.io/. Do not use for any other purpose." -const txsOnPage = 30 +const txsOnPage = 25 const txsInAPI = 1000 // PublicServer is a handle to public http server @@ -272,6 +272,8 @@ const ( errorTpl = tpl(iota) txTpl addressTpl + + tplCount ) type TemplateData struct { @@ -281,6 +283,10 @@ type TemplateData struct { AddrStr string Tx *api.Tx Error *api.ApiError + Page int + PrevPage int + NextPage int + PagingRange []int } func parseTemplates() []*template.Template { @@ -290,10 +296,10 @@ func parseTemplates() []*template.Template { "setTxToTemplateData": setTxToTemplateData, "stringInSlice": stringInSlice, } - t := make([]*template.Template, 3) - t[errorTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) + t := make([]*template.Template, tplCount) + t[errorTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) - t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/base.html")) + t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) return t } @@ -346,9 +352,65 @@ func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, err data := s.newTemplateData() data.AddrStr = address.AddrStr data.Address = address + data.Page = address.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) return addressTpl, data, nil } +func getPagingRange(page int, total int) ([]int, int, int) { + if total < 2 { + return nil, 0, 0 + } + pp, np := page-1, page+1 + if np > total { + np = total + } + if pp < 1 { + pp = 1 + } + r := make([]int, 0, 8) + if total < 6 { + for i := 1; i <= total; i++ { + r = append(r, i) + } + } else { + r = append(r, 1) + if page > 3 { + r = append(r, 0) + } + if pp == 1 { + if page == 1 { + r = append(r, np) + r = append(r, np+1) + r = append(r, np+2) + } else { + r = append(r, page) + r = append(r, np) + r = append(r, np+1) + } + } else if np == total { + if page == total { + r = append(r, pp-2) + r = append(r, pp-1) + r = append(r, pp) + } else { + r = append(r, pp-1) + r = append(r, pp) + r = append(r, page) + } + } else { + r = append(r, pp) + r = append(r, page) + r = append(r, np) + } + if page <= total-3 { + r = append(r, 0) + } + r = append(r, total) + } + return r, pp, np +} + type resAboutBlockbookPublic struct { Coin string `json:"coin"` Host string `json:"host"` diff --git a/static/css/main.css b/static/css/main.css index e2905211..f2ad1378 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -192,4 +192,38 @@ h3 { .navbar-nav .nav-link { padding-right: 0; padding-left: .25rem; +} + +.h-container { + display: -webkit-box; + display: -ms-flexbox; + -ms-flex-wrap: wrap; + flex-wrap: wrap; +} + +.h-container-6 { + width: 50%; + margin: 0; +} + +.h-container ul { + margin: 0; +} + +.page-link { + color: #428bca; +} + +.page-text { + display: block; + padding: .5rem .3rem; + line-height: 1.25; +} + +.page-link { + color: #428bca; +} + +.page-item.active .page-link { + background-color: #428bca; } \ No newline at end of file diff --git a/static/templates/address.html b/static/templates/address.html index 3a457703..b9100cb7 100644 --- a/static/templates/address.html +++ b/static/templates/address.html @@ -28,7 +28,7 @@
-{{if $addr.UnconfirmedTxApperances}} +{{- if $addr.UnconfirmedTxApperances -}}

Unconfirmed

@@ -44,9 +44,13 @@
-{{end}} {{if $addr.Transactions}} -

Transactions

-
- {{range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data }}{{end}} +{{- end}}{{if $addr.Transactions -}} +
+

Transactions

+
-{{end}} {{end}} \ No newline at end of file +
+ {{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}} +
+ +{{end}}{{end}} \ No newline at end of file diff --git a/static/templates/paging.html b/static/templates/paging.html new file mode 100644 index 00000000..fa9e7245 --- /dev/null +++ b/static/templates/paging.html @@ -0,0 +1,11 @@ +{{- define "paging"}}{{$data := . -}}{{if $data.PagingRange -}} +
    +
  • <
    • + {{- range $p := $data.PagingRange -}} +
  • + {{- if $p}}{{$p}} + {{- else -}}...{{- end -}} +
    • {{- end -}} +
  • >
    • +
+{{- end -}}{{- end -}} \ No newline at end of file From 78300b998e2403bfe2eb1e8cbb76a2c7744142bc Mon Sep 17 00:00:00 2001 From: Jakub Matys Date: Thu, 6 Sep 2018 11:45:13 +0200 Subject: [PATCH 058/123] Added 5min timeout to wait for service to stop --- build/templates/backend/debian/service | 1 + build/templates/blockbook/debian/service | 1 + 2 files changed, 2 insertions(+) diff --git a/build/templates/backend/debian/service b/build/templates/backend/debian/service index b494c22c..54473b3b 100644 --- a/build/templates/backend/debian/service +++ b/build/templates/backend/debian/service @@ -7,6 +7,7 @@ After=network.target ExecStart={{template "Backend.ExecCommandTemplate" .}} User={{.Backend.SystemUser}} Restart=on-failure +TimeoutStopSec=300 WorkingDirectory={{.Env.BackendInstallPath}}/{{.Coin.Alias}} {{if eq .Backend.ServiceType "forking" -}} Type=forking diff --git a/build/templates/blockbook/debian/service b/build/templates/blockbook/debian/service index 5bf291bc..e3541215 100644 --- a/build/templates/blockbook/debian/service +++ b/build/templates/blockbook/debian/service @@ -9,6 +9,7 @@ ExecStart={{.Env.BlockbookInstallPath}}/{{.Coin.Alias}}/bin/blockbook -blockchai User={{.Blockbook.SystemUser}} Type=simple Restart=on-failure +TimeoutStopSec=300 WorkingDirectory={{.Env.BlockbookInstallPath}}/{{.Coin.Alias}} # Resource limits From 8f5835a94d687e42c78adfb257d7a242c71fccd9 Mon Sep 17 00:00:00 2001 From: Jakub Matys Date: Thu, 6 Sep 2018 12:02:31 +0200 Subject: [PATCH 059/123] Set user-friendly name of Blockbook's process --- Gopkg.lock | 8 +++++++- blockbook.go | 28 +++++++++++++++++----------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 8618ddd9..d2eb4eca 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -67,6 +67,12 @@ revision = "1d4478f51bed434f1dadf96dcd9b43aabac66795" version = "v1.7" +[[projects]] + branch = "master" + name = "github.com/erikdubbelboer/gspt" + packages = ["."] + revision = "e39e726e09cc23d1ccf13b36ce10dbdb4a4510e0" + [[projects]] name = "github.com/ethereum/go-ethereum" packages = [".","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","ethdb","log","metrics","params","rlp","rpc","trie"] @@ -244,6 +250,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "03817720a4b1b5011cc64018a1344ed3fb4d8228af41c86856069f57851b3819" + inputs-digest = "98fb9a6252acd86d1c0a4b09a42e34673c2ccd6327caf49ed85bcee573601d95" solver-name = "gps-cdcl" solver-version = 1 diff --git a/blockbook.go b/blockbook.go index f3f91b31..a02ed158 100644 --- a/blockbook.go +++ b/blockbook.go @@ -1,10 +1,17 @@ package main import ( + "blockbook/bchain" + "blockbook/bchain/coins" + "blockbook/common" + "blockbook/db" + "blockbook/server" "context" "flag" "log" "math/rand" + "net/http" + _ "net/http/pprof" "os" "os/signal" "strings" @@ -12,18 +19,9 @@ import ( "syscall" "time" - "github.com/juju/errors" - - "blockbook/bchain" - "blockbook/bchain/coins" - "blockbook/common" - "blockbook/db" - "blockbook/server" - + "github.com/erikdubbelboer/gspt" "github.com/golang/glog" - - "net/http" - _ "net/http/pprof" + "github.com/juju/errors" ) // debounce too close requests for resync @@ -156,6 +154,8 @@ func main() { glog.Fatal("config: ", err) } + gspt.SetProcTitle("blockbook-" + normalizeName(coin)) + metrics, err := common.GetMetrics(coin) if err != nil { glog.Fatal("metrics: ", err) @@ -508,3 +508,9 @@ func printResult(txid string, vout uint32, isOutput bool) error { glog.Info(txid, vout, isOutput) return nil } + +func normalizeName(s string) string { + s = strings.ToLower(s) + s = strings.Replace(s, " ", "-", -1) + return s +} From a130318601cd67b61554a8b803be0ddbe294b6f9 Mon Sep 17 00:00:00 2001 From: Jakub Matys Date: Thu, 6 Sep 2018 08:56:29 +0200 Subject: [PATCH 060/123] Use forked version of btcutil --- Gopkg.lock | 18 ++++-- bchain/coins/bch/bcashparser.go | 8 +-- bchain/coins/bch/bcashrpc.go | 2 +- bchain/coins/btc/bitcoinparser.go | 6 +- bchain/coins/btg/bgoldparser.go | 6 +- bchain/coins/dash/dashparser.go | 14 ++--- bchain/coins/dogecoin/dogecoinparser.go | 6 +- bchain/coins/litecoin/litecoinparser.go | 10 +-- bchain/coins/monacoin/monacoinparser.go | 10 +-- bchain/coins/namecoin/namecoinparser.go | 6 +- bchain/coins/utils/address.go | 42 ------------- bchain/coins/vertcoin/vertcoinparser.go | 10 +-- bchain/coins/zec/zcashparser.go | 81 ++++++++++++++----------- bchain/coins/zec/zcashparser_test.go | 54 +++++++++++++++-- bchain/coins/zec/zcashrpc.go | 2 +- bchain/coins/zec/zcashrpc_test.go | 22 ++++--- 16 files changed, 159 insertions(+), 138 deletions(-) delete mode 100644 bchain/coins/utils/address.go diff --git a/Gopkg.lock b/Gopkg.lock index 8618ddd9..92e073b8 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -49,12 +49,6 @@ packages = ["ripemd160"] revision = "53f62d9b43e87a6c56975cf862af7edf33a8d0df" -[[projects]] - branch = "master" - name = "github.com/cpacia/bchutil" - packages = ["."] - revision = "12e86f41eb040d3b85b5d8e3a3a4bed035517c52" - [[projects]] name = "github.com/dchest/blake256" packages = ["."] @@ -121,6 +115,18 @@ revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" version = "v1.2.0" +[[projects]] + branch = "master" + name = "github.com/jakm/bchutil" + packages = ["."] + revision = "9e4fc13b082c87967b0befbcbad6fe5a5aa5ac1a" + +[[projects]] + branch = "master" + name = "github.com/jakm/btcutil" + packages = [".","base58","bech32","chaincfg","txscript"] + revision = "a45c5a6a9cb32f0caecb14a4e5a8f82640be1f39" + [[projects]] branch = "master" name = "github.com/juju/errors" diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 62f4c6b2..5977e2fd 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -5,10 +5,10 @@ import ( "blockbook/bchain/coins/btc" "fmt" - "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/txscript" - "github.com/btcsuite/btcutil" - "github.com/cpacia/bchutil" + "github.com/jakm/bchutil" + "github.com/jakm/btcutil" + "github.com/jakm/btcutil/chaincfg" + "github.com/jakm/btcutil/txscript" "github.com/schancel/cashaddr-converter/address" ) diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 5dd9b141..cbf4b94f 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -7,8 +7,8 @@ import ( "encoding/json" "math/big" - "github.com/cpacia/bchutil" "github.com/golang/glog" + "github.com/jakm/bchutil" "github.com/juju/errors" ) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index a4745be0..8dc08c47 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -9,10 +9,10 @@ import ( vlq "github.com/bsm/go-vlq" "github.com/btcsuite/btcd/blockchain" - "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" + "github.com/jakm/btcutil" + "github.com/jakm/btcutil/chaincfg" + "github.com/jakm/btcutil/txscript" ) // OutputScriptToAddressesFunc converts ScriptPubKey to bitcoin addresses diff --git a/bchain/coins/btg/bgoldparser.go b/bchain/coins/btg/bgoldparser.go index 67d2c7ef..ab4e38b5 100644 --- a/bchain/coins/btg/bgoldparser.go +++ b/bchain/coins/btg/bgoldparser.go @@ -7,9 +7,9 @@ import ( "bytes" "io" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -27,8 +27,8 @@ func init() { MainNetParams.Net = MainnetMagic // Address encoding magics - MainNetParams.PubKeyHashAddrID = 38 // base58 prefix: G - MainNetParams.ScriptHashAddrID = 23 // base58 prefix: A + MainNetParams.PubKeyHashAddrID = []byte{38} // base58 prefix: G + MainNetParams.ScriptHashAddrID = []byte{23} // base58 prefix: A TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic diff --git a/bchain/coins/dash/dashparser.go b/bchain/coins/dash/dashparser.go index 5590cc87..9b93e41b 100644 --- a/bchain/coins/dash/dashparser.go +++ b/bchain/coins/dash/dashparser.go @@ -3,8 +3,8 @@ package dash import ( "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -24,22 +24,22 @@ func init() { MainNetParams.Net = MainnetMagic // Address encoding magics - MainNetParams.PubKeyHashAddrID = 76 // base58 prefix: X - MainNetParams.ScriptHashAddrID = 16 // base58 prefix: 7 + MainNetParams.PubKeyHashAddrID = []byte{76} // base58 prefix: X + MainNetParams.ScriptHashAddrID = []byte{16} // base58 prefix: 7 TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic // Address encoding magics - TestNetParams.PubKeyHashAddrID = 140 // base58 prefix: y - TestNetParams.ScriptHashAddrID = 19 // base58 prefix: 8 or 9 + TestNetParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + TestNetParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 RegtestParams = chaincfg.RegressionNetParams RegtestParams.Net = RegtestMagic // Address encoding magics - RegtestParams.PubKeyHashAddrID = 140 // base58 prefix: y - RegtestParams.ScriptHashAddrID = 19 // base58 prefix: 8 or 9 + RegtestParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + RegtestParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 err := chaincfg.Register(&MainNetParams) if err == nil { diff --git a/bchain/coins/dogecoin/dogecoinparser.go b/bchain/coins/dogecoin/dogecoinparser.go index ac7ea94b..6975c688 100644 --- a/bchain/coins/dogecoin/dogecoinparser.go +++ b/bchain/coins/dogecoin/dogecoinparser.go @@ -6,8 +6,8 @@ import ( "blockbook/bchain/coins/utils" "bytes" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -21,8 +21,8 @@ var ( func init() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 30 - MainNetParams.ScriptHashAddrID = 22 + MainNetParams.PubKeyHashAddrID = []byte{30} + MainNetParams.ScriptHashAddrID = []byte{22} err := chaincfg.Register(&MainNetParams) if err != nil { diff --git a/bchain/coins/litecoin/litecoinparser.go b/bchain/coins/litecoin/litecoinparser.go index 3cb06f34..d81faa4f 100644 --- a/bchain/coins/litecoin/litecoinparser.go +++ b/bchain/coins/litecoin/litecoinparser.go @@ -3,8 +3,8 @@ package litecoin import ( "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -21,14 +21,14 @@ var ( func init() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 48 - MainNetParams.ScriptHashAddrID = 50 + MainNetParams.PubKeyHashAddrID = []byte{48} + MainNetParams.ScriptHashAddrID = []byte{50} MainNetParams.Bech32HRPSegwit = "ltc" TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic - TestNetParams.PubKeyHashAddrID = 111 - TestNetParams.ScriptHashAddrID = 58 + TestNetParams.PubKeyHashAddrID = []byte{111} + TestNetParams.ScriptHashAddrID = []byte{58} TestNetParams.Bech32HRPSegwit = "tltc" err := chaincfg.Register(&MainNetParams) diff --git a/bchain/coins/monacoin/monacoinparser.go b/bchain/coins/monacoin/monacoinparser.go index 4a05bdf0..ac082016 100644 --- a/bchain/coins/monacoin/monacoinparser.go +++ b/bchain/coins/monacoin/monacoinparser.go @@ -4,8 +4,8 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" monacoinCfg "github.com/wakiyamap/monad/chaincfg" "github.com/wakiyamap/monad/txscript" monacoinWire "github.com/wakiyamap/monad/wire" @@ -29,8 +29,8 @@ var ( func init() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 50 - MainNetParams.ScriptHashAddrID = 55 + MainNetParams.PubKeyHashAddrID = []byte{50} + MainNetParams.ScriptHashAddrID = []byte{55} MainNetParams.Bech32HRPSegwit = "mona" MonaMainParams = monacoinCfg.MainNetParams MonaMainParams.Net = MonaMainMagic @@ -40,8 +40,8 @@ func init() { TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic - TestNetParams.PubKeyHashAddrID = 111 - TestNetParams.ScriptHashAddrID = 117 + TestNetParams.PubKeyHashAddrID = []byte{111} + TestNetParams.ScriptHashAddrID = []byte{117} TestNetParams.Bech32HRPSegwit = "tmona" MonaTestParams = monacoinCfg.TestNet4Params MonaTestParams.Net = MonaTestMagic diff --git a/bchain/coins/namecoin/namecoinparser.go b/bchain/coins/namecoin/namecoinparser.go index 344dbeb4..270f1ed6 100644 --- a/bchain/coins/namecoin/namecoinparser.go +++ b/bchain/coins/namecoin/namecoinparser.go @@ -6,8 +6,8 @@ import ( "blockbook/bchain/coins/utils" "bytes" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -21,8 +21,8 @@ var ( func init() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 52 - MainNetParams.ScriptHashAddrID = 13 + MainNetParams.PubKeyHashAddrID = []byte{52} + MainNetParams.ScriptHashAddrID = []byte{13} err := chaincfg.Register(&MainNetParams) if err != nil { diff --git a/bchain/coins/utils/address.go b/bchain/coins/utils/address.go deleted file mode 100644 index 9fb2708e..00000000 --- a/bchain/coins/utils/address.go +++ /dev/null @@ -1,42 +0,0 @@ -package utils - -import ( - "crypto/sha256" - "errors" - - "github.com/btcsuite/btcutil/base58" -) - -var ( - // ErrChecksumMismatch describes an error where decoding failed due - // to a bad checksum. - ErrChecksumMismatch = errors.New("checksum mismatch") - - // ErrInvalidFormat describes an error where decoding failed due to invalid version - ErrInvalidFormat = errors.New("invalid format: version and/or checksum bytes missing") -) - -// checksum: first four bytes of sha256^2 -func checksum(input []byte) (cksum [4]byte) { - h := sha256.Sum256(input) - h2 := sha256.Sum256(h[:]) - copy(cksum[:], h2[:4]) - return -} - -// CheckDecode decodes a string that was encoded with CheckEncode and verifies the checksum. -func CheckDecode(input string) (result []byte, version []byte, err error) { - decoded := base58.Decode(input) - if len(decoded) < 5 { - return nil, nil, ErrInvalidFormat - } - version = append(version, decoded[0:2]...) - var cksum [4]byte - copy(cksum[:], decoded[len(decoded)-4:]) - if checksum(decoded[:len(decoded)-4]) != cksum { - return nil, nil, ErrChecksumMismatch - } - payload := decoded[2 : len(decoded)-4] - result = append(result, payload...) - return -} diff --git a/bchain/coins/vertcoin/vertcoinparser.go b/bchain/coins/vertcoin/vertcoinparser.go index 6f33a9ae..59fdbdcf 100644 --- a/bchain/coins/vertcoin/vertcoinparser.go +++ b/bchain/coins/vertcoin/vertcoinparser.go @@ -3,8 +3,8 @@ package vertcoin import ( "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -21,14 +21,14 @@ var ( func init() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 71 - MainNetParams.ScriptHashAddrID = 5 + MainNetParams.PubKeyHashAddrID = []byte{71} + MainNetParams.ScriptHashAddrID = []byte{5} MainNetParams.Bech32HRPSegwit = "vtc" TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic - TestNetParams.PubKeyHashAddrID = 74 - TestNetParams.ScriptHashAddrID = 196 + TestNetParams.PubKeyHashAddrID = []byte{74} + TestNetParams.ScriptHashAddrID = []byte{196} TestNetParams.Bech32HRPSegwit = "tvtc" err := chaincfg.Register(&MainNetParams) diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index 7b7e6e20..c6452fc7 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -3,11 +3,9 @@ package zec import ( "blockbook/bchain" "blockbook/bchain/coins/btc" - "blockbook/bchain/coins/utils" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" - "github.com/juju/errors" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -16,18 +14,48 @@ const ( RegtestMagic wire.BitcoinNet = 0x5f3fe8aa ) +var ( + MainNetParams chaincfg.Params + TestNetParams chaincfg.Params +) + +func init() { + MainNetParams = chaincfg.MainNetParams + MainNetParams.Net = MainnetMagic + + // Address encoding magics + MainNetParams.AddressMagicLen = 2 + MainNetParams.PubKeyHashAddrID = []byte{0x1C, 0xB8} // base58 prefix: t1 + MainNetParams.ScriptHashAddrID = []byte{0x1C, 0xBD} // base58 prefix: t3 + + TestNetParams = chaincfg.TestNet3Params + TestNetParams.Net = TestnetMagic + + // Address encoding magics + TestNetParams.AddressMagicLen = 2 + TestNetParams.PubKeyHashAddrID = []byte{0x1D, 0x25} // base58 prefix: tm + TestNetParams.ScriptHashAddrID = []byte{0x1C, 0xBA} // base58 prefix: t2 + + err := chaincfg.Register(&MainNetParams) + if err == nil { + err = chaincfg.Register(&TestNetParams) + } + if err != nil { + panic(err) + } +} + // ZCashParser handle type ZCashParser struct { - *bchain.BaseParser + *btc.BitcoinParser + baseparser *bchain.BaseParser } // NewZCashParser returns new ZCashParser instance -func NewZCashParser(c *btc.Configuration) *ZCashParser { +func NewZCashParser(params *chaincfg.Params, c *btc.Configuration) *ZCashParser { return &ZCashParser{ - &bchain.BaseParser{ - BlockAddressesToKeep: c.BlockAddressesToKeep, - AmountDecimalPoint: 8, - }, + BitcoinParser: btc.NewBitcoinParser(params, c), + baseparser: &bchain.BaseParser{}, } } @@ -38,42 +66,23 @@ func GetChainParams(chain string) *chaincfg.Params { var params *chaincfg.Params switch chain { case "test": - params = &chaincfg.TestNet3Params - params.Net = TestnetMagic + return &TestNetParams case "regtest": params = &chaincfg.RegressionNetParams params.Net = RegtestMagic default: - params = &chaincfg.MainNetParams - params.Net = MainnetMagic + return &MainNetParams } return params } -// GetAddrDescFromVout returns internal address representation of given transaction output -func (p *ZCashParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { - if len(output.ScriptPubKey.Addresses) != 1 { - return nil, nil - } - hash, _, err := utils.CheckDecode(output.ScriptPubKey.Addresses[0]) - return hash, err +// PackTx packs transaction to byte array using protobuf +func (p *ZCashParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { + return p.baseparser.PackTx(tx, height, blockTime) } -// GetAddrDescFromAddress returns internal address representation of given address -func (p *ZCashParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { - hash, _, err := utils.CheckDecode(address) - return hash, err -} - -// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable -func (p *ZCashParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { - // TODO implement - return nil, false, errors.New("GetAddressesFromAddrDesc: not implemented") -} - -// GetScriptFromAddrDesc returns output script for given address descriptor -func (p *ZCashParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]byte, error) { - // TODO implement - return nil, errors.New("GetScriptFromAddrDesc: not implemented") +// UnpackTx unpacks transaction from protobuf byte array +func (p *ZCashParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { + return p.baseparser.UnpackTx(buf) } diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index c5e02b5c..ebaf5135 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -5,6 +5,7 @@ package zec import ( "blockbook/bchain" "blockbook/bchain/coins/btc" + "bytes" "encoding/hex" "math/big" "reflect" @@ -90,6 +91,51 @@ func init() { } } +func TestGetAddrDesc(t *testing.T) { + type args struct { + tx bchain.Tx + parser *ZCashParser + } + tests := []struct { + name string + args args + }{ + { + name: "zec-1", + args: args{ + tx: testTx1, + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), + }, + }, + { + name: "zec-2", + args: args{ + tx: testTx2, + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for n, vout := range tt.args.tx.Vout { + got1, err := tt.args.parser.GetAddrDescFromVout(&vout) + if err != nil { + t.Errorf("getAddrDescFromVout() error = %v, vout = %d", err, n) + return + } + got2, err := tt.args.parser.GetAddrDescFromAddress(vout.ScriptPubKey.Addresses[0]) + if err != nil { + t.Errorf("getAddrDescFromAddress() error = %v, vout = %d", err, n) + return + } + if !bytes.Equal(got1, got2) { + t.Errorf("Address descriptors mismatch: got1 = %v, got2 = %v", got1, got2) + } + } + }) + } +} + func TestPackTx(t *testing.T) { type args struct { tx bchain.Tx @@ -109,7 +155,7 @@ func TestPackTx(t *testing.T) { tx: testTx1, height: 292272, blockTime: 1521645728, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: testTxPacked1, wantErr: false, @@ -120,7 +166,7 @@ func TestPackTx(t *testing.T) { tx: testTx2, height: 292217, blockTime: 1521637604, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: testTxPacked2, wantErr: false, @@ -157,7 +203,7 @@ func TestUnpackTx(t *testing.T) { name: "zec-1", args: args{ packedTx: testTxPacked1, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: &testTx1, want1: 292272, @@ -167,7 +213,7 @@ func TestUnpackTx(t *testing.T) { name: "zec-2", args: args{ packedTx: testTxPacked2, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: &testTx2, want1: 292217, diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index b2c45fe5..6c9b023b 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -35,7 +35,7 @@ func (z *ZCashRPC) Initialize() error { params := GetChainParams(chainName) - z.Parser = NewZCashParser(z.ChainConfig) + z.Parser = NewZCashParser(params, z.ChainConfig) // parameters for getInfo request if params.Net == MainnetMagic { diff --git a/bchain/coins/zec/zcashrpc_test.go b/bchain/coins/zec/zcashrpc_test.go index f98520c6..be6be633 100644 --- a/bchain/coins/zec/zcashrpc_test.go +++ b/bchain/coins/zec/zcashrpc_test.go @@ -11,15 +11,17 @@ import ( "testing" ) -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewZCashRPC(cfg, nil) - if err != nil { - return nil, err +func getRPCClient(chain string) func(cfg json.RawMessage) (bchain.BlockChain, error) { + return func(cfg json.RawMessage) (bchain.BlockChain, error) { + c, err := NewZCashRPC(cfg, nil) + if err != nil { + return nil, err + } + cli := c.(*ZCashRPC) + cli.Parser = NewZCashParser(GetChainParams(chain), cli.ChainConfig) + cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) + return cli, nil } - cli := c.(*ZCashRPC) - cli.Parser = NewZCashParser(cli.ChainConfig) - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil } var tests struct { @@ -30,14 +32,14 @@ var tests struct { func TestMain(m *testing.M) { flag.Parse() - t, err := rpc.NewTest("Zcash", getRPCClient) + t, err := rpc.NewTest("Zcash", getRPCClient("main")) if err != nil { panic(err) } tests.mainnet = t - t, err = rpc.NewTest("Zcash Testnet", getRPCClient) + t, err = rpc.NewTest("Zcash Testnet", getRPCClient("test")) if err != nil { panic(err) } From 6d15e429c7cc27605e03c4efa6d253e4e6b96cd1 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 11 Sep 2018 16:07:39 +0200 Subject: [PATCH 061/123] Improve OP_RETURN decoding --- bchain/coins/btc/bitcoinparser.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 02814f89..41f9f2ad 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -87,8 +87,22 @@ func (p *BitcoinParser) addressToOutputScript(address string) ([]byte, error) { // TryParseOPReturn tries to process OP_RETURN script and return its string representation func TryParseOPReturn(script []byte) string { if len(script) > 1 && script[0] == txscript.OP_RETURN { - l := int(script[1]) - data := script[2:] + // trying 2 variants of OP_RETURN data + // 1) OP_RETURN OP_PUSHDATA1 + // 2) OP_RETURN + var data []byte + var l int + if script[1] == txscript.OP_PUSHDATA1 && len(script) > 2 { + l = int(script[2]) + data = script[3:] + if l != len(data) { + l = int(script[1]) + data = script[2:] + } + } else { + l = int(script[1]) + data = script[2:] + } if l == len(data) { isASCII := true for _, c := range data { @@ -101,7 +115,7 @@ func TryParseOPReturn(script []byte) string { if isASCII { ed = "(" + string(data) + ")" } else { - ed = hex.EncodeToString([]byte{byte(l)}) + " " + hex.EncodeToString(data) + ed = hex.EncodeToString(data) } return "OP_RETURN " + ed } From 402eb92c5e3459a5f2d4482a4374349a5e860e10 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 11 Sep 2018 16:10:59 +0200 Subject: [PATCH 062/123] Use internal explorer for bcash and zcash --- configs/coins/bcash.json | 2 +- configs/coins/bcash_testnet.json | 2 +- configs/coins/zcash.json | 2 +- configs/coins/zcash_testnet.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 78c74336..aee70727 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -44,7 +44,7 @@ "system_user": "blockbook-bcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://bitcoincash.blockexplorer.com", + "explorer_url": "/explorer", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index af58f66a..9b16e9a5 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -44,7 +44,7 @@ "system_user": "blockbook-bcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://bitcoincash.blockexplorer.com", + "explorer_url": "/explorer", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index ab65b246..1d697582 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -47,7 +47,7 @@ "system_user": "blockbook-zcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://zcash.blockexplorer.com/", + "explorer_url": "/explorer", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index c2793c96..70a3dbd2 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -47,7 +47,7 @@ "system_user": "blockbook-zcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://explorer.testnet.z.cash/", + "explorer_url": "/explorer", "additional_params": "", "block_chain": { "parse": true, From 2c62d4d2488d3ee2958d3f6b9167ec0c6d125c68 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 11 Sep 2018 16:22:28 +0200 Subject: [PATCH 063/123] Fix OP_RETURN parser unit tests --- bchain/coins/bch/bcashparser_test.go | 2 +- bchain/coins/btc/bitcoinparser_test.go | 9 ++++++++- bchain/coins/monacoin/monacoinparser_test.go | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index d9e38ac7..71464b57 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -157,7 +157,7 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { { name: "OP_RETURN hex", parser: mainParserCashAddr, - addresses: []string{"OP_RETURN 07 2020f1686f6a20"}, + addresses: []string{"OP_RETURN 2020f1686f6a20"}, searchable: false, hex: "6a072020f1686f6a20", wantErr: false, diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 2350f3ef..80245180 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -108,10 +108,17 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { want2: false, wantErr: false, }, + { + name: "OP_RETURN OP_PUSHDATA1 ascii", + args: args{script: "6a4c0b446c6f7568792074657874"}, + want: []string{"OP_RETURN (Dlouhy text)"}, + want2: false, + wantErr: false, + }, { name: "OP_RETURN hex", args: args{script: "6a072020f1686f6a20"}, - want: []string{"OP_RETURN 07 2020f1686f6a20"}, + want: []string{"OP_RETURN 2020f1686f6a20"}, want2: false, wantErr: false, }, diff --git a/bchain/coins/monacoin/monacoinparser_test.go b/bchain/coins/monacoin/monacoinparser_test.go index 3b9e759e..7c95d9a2 100644 --- a/bchain/coins/monacoin/monacoinparser_test.go +++ b/bchain/coins/monacoin/monacoinparser_test.go @@ -170,7 +170,7 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { { name: "OP_RETURN hex", args: args{script: "6a072020f1686f6a20"}, - want: []string{"OP_RETURN 07 2020f1686f6a20"}, + want: []string{"OP_RETURN 2020f1686f6a20"}, want2: false, wantErr: false, }, From b5d1e5c94e06865a5ae47acbe4aa6ab45ab3b95a Mon Sep 17 00:00:00 2001 From: Jakub Matys Date: Tue, 11 Sep 2018 16:36:32 +0200 Subject: [PATCH 064/123] Integration tests are defined declaratively with single implementation --- bchain/coins/bch/bcashrpc_test.go | 141 ----- bchain/coins/blockchain.go | 44 +- bchain/coins/btc/bitcoinrpc_test.go | 142 ----- bchain/coins/dash/dashrpc_test.go | 129 ----- bchain/coins/dogecoin/dogecoinrpc_test.go | 70 --- bchain/coins/eth/ethrpc_test.go | 62 --- bchain/coins/litecoin/litecoinrpc_test.go | 72 --- bchain/coins/monacoin/monacoinrpc_test.go | 72 --- bchain/coins/namecoin/namecoinrpc_test.go | 86 ---- bchain/coins/vertcoin/vertcoinrpc_test.go | 84 --- bchain/coins/zec/zcashrpc_test.go | 130 ----- bchain/tests/rpc/config.json | 77 --- bchain/tests/rpc/data.go | 49 +- bchain/tests/rpc/rpc.go | 455 ---------------- bchain/tests/rpc/rpc_test.go | 484 ++++++++++++++++++ .../rpc/testdata/{Bcash.json => bcash.json} | 0 ...{Bcash_Testnet.json => bcash_testnet.json} | 0 .../testdata/{Bitcoin.json => bitcoin.json} | 0 ...coin_Testnet.json => bitcoin_testnet.json} | 0 .../rpc/testdata/{Dash.json => dash.json} | 0 .../{Dash_Testnet.json => dash_testnet.json} | 0 .../testdata/{Dogecoin.json => dogecoin.json} | 0 ...net.json => ethereum_testnet_ropsten.json} | 0 .../testdata/{Litecoin.json => litecoin.json} | 0 .../testdata/{Monacoin.json => monacoin.json} | 0 .../testdata/{Namecoin.json => namecoin.json} | 0 .../testdata/{Vertcoin.json => vertcoin.json} | 0 .../rpc/testdata/{Zcash.json => zcash.json} | 0 ...{Zcash_Testnet.json => zcash_testnet.json} | 0 build/docker/bin/Makefile | 15 +- build/templates/generate.go | 288 +---------- build/tools/templates.go | 288 +++++++++++ configs/coins/bcash.json | 4 + configs/coins/bcash_testnet.json | 4 + configs/coins/bitcoin.json | 4 + configs/coins/bitcoin_testnet.json | 4 + configs/coins/dash.json | 4 + configs/coins/dash_testnet.json | 4 + configs/coins/dogecoin.json | 3 + configs/coins/ethereum_testnet_ropsten.json | 3 + configs/coins/litecoin.json | 4 + configs/coins/monacoin.json | 4 + configs/coins/namecoin.json | 4 + configs/coins/vertcoin.json | 4 + configs/coins/zcash.json | 4 + configs/coins/zcash_testnet.json | 4 + 46 files changed, 876 insertions(+), 1866 deletions(-) delete mode 100644 bchain/coins/bch/bcashrpc_test.go delete mode 100644 bchain/coins/btc/bitcoinrpc_test.go delete mode 100644 bchain/coins/dash/dashrpc_test.go delete mode 100644 bchain/coins/dogecoin/dogecoinrpc_test.go delete mode 100644 bchain/coins/eth/ethrpc_test.go delete mode 100644 bchain/coins/litecoin/litecoinrpc_test.go delete mode 100644 bchain/coins/monacoin/monacoinrpc_test.go delete mode 100644 bchain/coins/namecoin/namecoinrpc_test.go delete mode 100644 bchain/coins/vertcoin/vertcoinrpc_test.go delete mode 100644 bchain/coins/zec/zcashrpc_test.go delete mode 100644 bchain/tests/rpc/config.json delete mode 100644 bchain/tests/rpc/rpc.go create mode 100644 bchain/tests/rpc/rpc_test.go rename bchain/tests/rpc/testdata/{Bcash.json => bcash.json} (100%) rename bchain/tests/rpc/testdata/{Bcash_Testnet.json => bcash_testnet.json} (100%) rename bchain/tests/rpc/testdata/{Bitcoin.json => bitcoin.json} (100%) rename bchain/tests/rpc/testdata/{Bitcoin_Testnet.json => bitcoin_testnet.json} (100%) rename bchain/tests/rpc/testdata/{Dash.json => dash.json} (100%) rename bchain/tests/rpc/testdata/{Dash_Testnet.json => dash_testnet.json} (100%) rename bchain/tests/rpc/testdata/{Dogecoin.json => dogecoin.json} (100%) rename bchain/tests/rpc/testdata/{Ethereum_Testnet.json => ethereum_testnet_ropsten.json} (100%) rename bchain/tests/rpc/testdata/{Litecoin.json => litecoin.json} (100%) rename bchain/tests/rpc/testdata/{Monacoin.json => monacoin.json} (100%) rename bchain/tests/rpc/testdata/{Namecoin.json => namecoin.json} (100%) rename bchain/tests/rpc/testdata/{Vertcoin.json => vertcoin.json} (100%) rename bchain/tests/rpc/testdata/{Zcash.json => zcash.json} (100%) rename bchain/tests/rpc/testdata/{Zcash_Testnet.json => zcash_testnet.json} (100%) create mode 100644 build/tools/templates.go diff --git a/bchain/coins/bch/bcashrpc_test.go b/bchain/coins/bch/bcashrpc_test.go deleted file mode 100644 index 7056125b..00000000 --- a/bchain/coins/bch/bcashrpc_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// +build integration - -package bch - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewBCashRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*BCashRPC) - cli.Parser, err = NewBCashParser(GetChainParams(chain), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - - t, err := rpc.NewTest("Bcash", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Bcash Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestBCashRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestBCashRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestBCashRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestBCashRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -func TestBCashRPC_MempoolSync(t *testing.T) { - tests.mainnet.TestMempoolSync(t) -} - -func TestBCashRPC_GetMempoolEntry(t *testing.T) { - tests.mainnet.TestGetMempoolEntry(t) -} - -func TestBCashRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestBCashRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestBCashRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestBCashRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestBCashRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestBCashTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestBCashTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestBCashTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestBCashTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestBCashTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestBCashTestnetRPC_GetMempoolEntry(t *testing.T) { - tests.testnet.TestGetMempoolEntry(t) -} - -func TestBCashTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestBCashTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestBCashTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.testnet.TestGetBestBlockHash(t) -} - -func TestBCashTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.testnet.TestGetBestBlockHeight(t) -} - -func TestBCashTestnetRPC_GetBlockHeader(t *testing.T) { - tests.testnet.TestGetBlockHeader(t) -} diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index b0be1d2b..3e3f6ae0 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -27,28 +27,28 @@ import ( type blockChainFactory func(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) -var blockChainFactories = make(map[string]blockChainFactory) +var BlockChainFactories = make(map[string]blockChainFactory) func init() { - blockChainFactories["Bitcoin"] = btc.NewBitcoinRPC - blockChainFactories["Testnet"] = btc.NewBitcoinRPC - blockChainFactories["Zcash"] = zec.NewZCashRPC - blockChainFactories["Zcash Testnet"] = zec.NewZCashRPC - blockChainFactories["Ethereum"] = eth.NewEthereumRPC - blockChainFactories["Ethereum Testnet Ropsten"] = eth.NewEthereumRPC - blockChainFactories["Bcash"] = bch.NewBCashRPC - blockChainFactories["Bcash Testnet"] = bch.NewBCashRPC - blockChainFactories["Bgold"] = btg.NewBGoldRPC - blockChainFactories["Dash"] = dash.NewDashRPC - blockChainFactories["Dash Testnet"] = dash.NewDashRPC - blockChainFactories["Litecoin"] = litecoin.NewLitecoinRPC - blockChainFactories["Litecoin Testnet"] = litecoin.NewLitecoinRPC - blockChainFactories["Dogecoin"] = dogecoin.NewDogecoinRPC - blockChainFactories["Vertcoin"] = vertcoin.NewVertcoinRPC - blockChainFactories["Vertcoin Testnet"] = vertcoin.NewVertcoinRPC - blockChainFactories["Namecoin"] = namecoin.NewNamecoinRPC - blockChainFactories["Monacoin"] = monacoin.NewMonacoinRPC - blockChainFactories["Monacoin Testnet"] = monacoin.NewMonacoinRPC + BlockChainFactories["Bitcoin"] = btc.NewBitcoinRPC + BlockChainFactories["Testnet"] = btc.NewBitcoinRPC + BlockChainFactories["Zcash"] = zec.NewZCashRPC + BlockChainFactories["Zcash Testnet"] = zec.NewZCashRPC + BlockChainFactories["Ethereum"] = eth.NewEthereumRPC + BlockChainFactories["Ethereum Testnet Ropsten"] = eth.NewEthereumRPC + BlockChainFactories["Bcash"] = bch.NewBCashRPC + BlockChainFactories["Bcash Testnet"] = bch.NewBCashRPC + BlockChainFactories["Bgold"] = btg.NewBGoldRPC + BlockChainFactories["Dash"] = dash.NewDashRPC + BlockChainFactories["Dash Testnet"] = dash.NewDashRPC + BlockChainFactories["Litecoin"] = litecoin.NewLitecoinRPC + BlockChainFactories["Litecoin Testnet"] = litecoin.NewLitecoinRPC + BlockChainFactories["Dogecoin"] = dogecoin.NewDogecoinRPC + BlockChainFactories["Vertcoin"] = vertcoin.NewVertcoinRPC + BlockChainFactories["Vertcoin Testnet"] = vertcoin.NewVertcoinRPC + BlockChainFactories["Namecoin"] = namecoin.NewNamecoinRPC + BlockChainFactories["Monacoin"] = monacoin.NewMonacoinRPC + BlockChainFactories["Monacoin Testnet"] = monacoin.NewMonacoinRPC } // GetCoinNameFromConfig gets coin name and coin shortcut from config file @@ -79,9 +79,9 @@ func NewBlockChain(coin string, configfile string, pushHandler func(bchain.Notif if err != nil { return nil, errors.Annotatef(err, "Error parsing file %v", configfile) } - bcf, ok := blockChainFactories[coin] + bcf, ok := BlockChainFactories[coin] if !ok { - return nil, errors.New(fmt.Sprint("Unsupported coin '", coin, "'. Must be one of ", reflect.ValueOf(blockChainFactories).MapKeys())) + return nil, errors.New(fmt.Sprint("Unsupported coin '", coin, "'. Must be one of ", reflect.ValueOf(BlockChainFactories).MapKeys())) } bc, err := bcf(config, pushHandler) if err != nil { diff --git a/bchain/coins/btc/bitcoinrpc_test.go b/bchain/coins/btc/bitcoinrpc_test.go deleted file mode 100644 index 882a1bd7..00000000 --- a/bchain/coins/btc/bitcoinrpc_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// +build integration - -package btc - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewBitcoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*BitcoinRPC) - cli.Parser = NewBitcoinParser(GetChainParams(chain), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Bitcoin", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Bitcoin Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestBitcoinRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestBitcoinRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestBitcoinRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestBitcoinRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -// FIXME -func TestBitcoinRPC_MempoolSync(t *testing.T) { - t.Skip("skipping test, run too long") - // tests.mainnet.TestMempoolSync(t) -} - -func TestBitcoinRPC_GetMempoolEntry(t *testing.T) { - tests.mainnet.TestGetMempoolEntry(t) -} - -func TestBitcoinRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestBitcoinRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestBitcoinRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestBitcoinRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestBitcoinRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestBitcoinTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestBitcoinTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestBitcoinTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestBitcoinTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestBitcoinTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestBitcoinTestnetRPC_GetMempoolEntry(t *testing.T) { - tests.testnet.TestGetMempoolEntry(t) -} - -func TestBitcoinTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestBitcoinTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestBitcoinTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.testnet.TestGetBestBlockHash(t) -} - -func TestBitcoinTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.testnet.TestGetBestBlockHeight(t) -} - -func TestBitcoinTestnetRPC_GetBlockHeader(t *testing.T) { - tests.testnet.TestGetBlockHeader(t) -} diff --git a/bchain/coins/dash/dashrpc_test.go b/bchain/coins/dash/dashrpc_test.go deleted file mode 100644 index 98662672..00000000 --- a/bchain/coins/dash/dashrpc_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// +build integration - -package dash - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewDashRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*DashRPC) - cli.Parser = NewDashParser(GetChainParams(chain), cli.ChainConfig) - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Dash", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Dash Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestDashRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestDashRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestDashRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestDashRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -func TestDashRPC_MempoolSync(t *testing.T) { - tests.mainnet.TestMempoolSync(t) -} - -func TestDashRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestDashRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestDashRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestDashRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestDashRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestDashTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestDashTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestDashTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestDashTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestDashTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestDashTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestDashTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestDashTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.testnet.TestGetBestBlockHash(t) -} - -func TestDashTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.testnet.TestGetBestBlockHeight(t) -} - -func TestDashTestnetRPC_GetBlockHeader(t *testing.T) { - tests.testnet.TestGetBlockHeader(t) -} diff --git a/bchain/coins/dogecoin/dogecoinrpc_test.go b/bchain/coins/dogecoin/dogecoinrpc_test.go deleted file mode 100644 index b9d3bbc4..00000000 --- a/bchain/coins/dogecoin/dogecoinrpc_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// +build integration - -package dogecoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewDogecoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*DogecoinRPC) - cli.Parser = NewDogecoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Dogecoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestDogecoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestDogecoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestDogecoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestDogecoinRPC_GetTransactionForMempool(t *testing.T) { - rpcTest.TestGetTransactionForMempool(t) -} - -func TestDogecoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestDogecoinRPC_EstimateSmartFee(t *testing.T) { - t.Skip("skipping test, unreliable") - // rpcTest.TestEstimateSmartFee(t) -} - -func TestDogecoinRPC_EstimateFee(t *testing.T) { - t.Skip("skipping test, unreliable") - // rpcTest.TestEstimateFee(t) -} diff --git a/bchain/coins/eth/ethrpc_test.go b/bchain/coins/eth/ethrpc_test.go deleted file mode 100644 index fab3349f..00000000 --- a/bchain/coins/eth/ethrpc_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// +build integration - -package eth - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewEthereumRPC(cfg, nil) - if err != nil { - return nil, err - } - return c, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Ethereum Testnet", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestEthRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestEthRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestEthRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestEthRPC_GetBestBlockHash(t *testing.T) { - rpcTest.TestGetBestBlockHash(t) -} - -func TestEthRPC_GetBestBlockHeight(t *testing.T) { - rpcTest.TestGetBestBlockHeight(t) -} - -func TestEthRPC_GetBlockHeader(t *testing.T) { - rpcTest.TestGetBlockHeader(t) -} - -func TestEthRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} diff --git a/bchain/coins/litecoin/litecoinrpc_test.go b/bchain/coins/litecoin/litecoinrpc_test.go deleted file mode 100644 index 33a67b91..00000000 --- a/bchain/coins/litecoin/litecoinrpc_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// +build integration - -package litecoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewLitecoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*LitecoinRPC) - cli.Parser = NewLitecoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Litecoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestLitecoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestLitecoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestLitecoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestLitecoinRPC_GetTransactionForMempool(t *testing.T) { - rpcTest.TestGetTransactionForMempool(t) -} - -func TestLitecoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestLitecoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestLitecoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestLitecoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} diff --git a/bchain/coins/monacoin/monacoinrpc_test.go b/bchain/coins/monacoin/monacoinrpc_test.go deleted file mode 100644 index 4a4a7f5c..00000000 --- a/bchain/coins/monacoin/monacoinrpc_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// +build integration - -package monacoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewMonacoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*MonacoinRPC) - cli.Parser = NewMonacoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Monacoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestMonacoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestMonacoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestMonacoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestMonacoinRPC_GetTransactionForMempool(t *testing.T) { - rpcTest.TestGetTransactionForMempool(t) -} - -func TestMonacoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestMonacoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestMonacoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestMonacoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} diff --git a/bchain/coins/namecoin/namecoinrpc_test.go b/bchain/coins/namecoin/namecoinrpc_test.go deleted file mode 100644 index d50ef581..00000000 --- a/bchain/coins/namecoin/namecoinrpc_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// +build integration - -package namecoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewNamecoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*NamecoinRPC) - cli.Parser = NewNamecoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Namecoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestNamecoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestNamecoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestNamecoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestNamecoinRPC_GetTransactionForMempool(t *testing.T) { - // extra opcodes (name_new, name_firstupdate, name_update) aren't supported, so some transactions - // in mempool can't be parsed correctly - t.Skipf("Skipped because of instability") -} - -func TestNamecoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestNamecoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestNamecoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestNamecoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} - -func TestNamecoinRPC_GetBestBlockHash(t *testing.T) { - rpcTest.TestGetBestBlockHash(t) -} - -func TestNamecoinRPC_GetBestBlockHeight(t *testing.T) { - rpcTest.TestGetBestBlockHeight(t) -} - -func TestNamecoinRPC_GetBlockHeader(t *testing.T) { - rpcTest.TestGetBlockHeader(t) -} diff --git a/bchain/coins/vertcoin/vertcoinrpc_test.go b/bchain/coins/vertcoin/vertcoinrpc_test.go deleted file mode 100644 index aa4c615e..00000000 --- a/bchain/coins/vertcoin/vertcoinrpc_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// +build integration - -package vertcoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewVertcoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*VertcoinRPC) - cli.Parser = NewVertcoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Vertcoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestVertcoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestVertcoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestVertcoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestVertcoinRPC_GetTransactionForMempool(t *testing.T) { - rpcTest.TestGetTransactionForMempool(t) -} - -func TestVertcoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestVertcoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestVertcoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestVertcoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} - -func TestVertcoinRPC_GetBestBlockHash(t *testing.T) { - rpcTest.TestGetBestBlockHash(t) -} - -func TestVertcoinRPC_GetBestBlockHeight(t *testing.T) { - rpcTest.TestGetBestBlockHeight(t) -} - -func TestVertcoinRPC_GetBlockHeader(t *testing.T) { - rpcTest.TestGetBlockHeader(t) -} diff --git a/bchain/coins/zec/zcashrpc_test.go b/bchain/coins/zec/zcashrpc_test.go deleted file mode 100644 index be6be633..00000000 --- a/bchain/coins/zec/zcashrpc_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// +build integration - -package zec - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(cfg json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewZCashRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*ZCashRPC) - cli.Parser = NewZCashParser(GetChainParams(chain), cli.ChainConfig) - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - - t, err := rpc.NewTest("Zcash", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Zcash Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestZCashRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestZCashRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestZCashRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestZCashRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -func TestZCashRPC_MempoolSync(t *testing.T) { - tests.mainnet.TestMempoolSync(t) -} - -func TestZCashRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestZCashRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestZCashRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestZCashRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestZCashRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestZCashTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestZCashTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestZCashTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestZCashTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestZCashTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestZCashTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestZCashTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestZCashTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestZCashTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestZCashTestnetRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} diff --git a/bchain/tests/rpc/config.json b/bchain/tests/rpc/config.json deleted file mode 100644 index 2e8be7a8..00000000 --- a/bchain/tests/rpc/config.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "Bitcoin": { - "url": "http://localhost:8030", - "user": "rpc", - "pass": "rpc" - }, - "Bcash": { - "url": "http://localhost:8031", - "user": "rpc", - "pass": "rpc" - }, - "Zcash": { - "url": "http://localhost:8032", - "user": "rpc", - "pass": "rpc" - }, - "Dash": { - "url": "http://localhost:8033", - "user": "rpc", - "pass": "rpc" - }, - "Litecoin": { - "url": "http://localhost:8034", - "user": "rpc", - "pass": "rpc" - }, - "Ethereum": { - "url": "ws://localhost:8036", - "user": null, - "pass": null - }, - "Dogecoin": { - "url": "http://localhost:8038", - "user": "rpc", - "pass": "rpcp" - }, - "Namecoin": { - "url": "http://localhost:8039", - "user": "rpc", - "pass": "rpc" - }, - "Vertcoin": { - "url": "http://localhost:8040", - "user": "rpc", - "pass": "rpc" - }, - "Monacoin": { - "url": "http://localhost:8041", - "user": "rpc", - "pass": "rpc" - }, - "Bitcoin Testnet": { - "url": "http://localhost:18030", - "user": "rpc", - "pass": "rpc" - }, - "Bcash Testnet": { - "url": "http://localhost:18031", - "user": "rpc", - "pass": "rpc" - }, - "Zcash Testnet": { - "url": "http://localhost:18032", - "user": "rpc", - "pass": "rpc" - }, - "Dash Testnet": { - "url": "http://localhost:18033", - "user": "rpc", - "pass": "rpc" - }, - "Ethereum Testnet": { - "url": "ws://localhost:18036", - "user": null, - "pass": null - } -} \ No newline at end of file diff --git a/bchain/tests/rpc/data.go b/bchain/tests/rpc/data.go index d855ed48..dbe8c1f8 100644 --- a/bchain/tests/rpc/data.go +++ b/bchain/tests/rpc/data.go @@ -4,12 +4,19 @@ import ( "blockbook/bchain" "encoding/json" "errors" - "fmt" "io/ioutil" "path/filepath" "strings" ) +type TestData struct { + BlockHeight uint32 `json:"blockHeight"` + BlockHash string `json:"blockHash"` + BlockTime int64 `json:"blockTime"` + BlockTxs []string `json:"blockTxs"` + TxDetails map[string]*bchain.Tx `json:"txDetails"` +} + func joinPathsWithCommonElement(p1, p2 string) (string, bool) { idx := strings.IndexRune(p2, filepath.Separator) if idx <= 0 { @@ -42,46 +49,6 @@ func readDataFile(dir, relDir, filename string) ([]byte, error) { return ioutil.ReadFile(path) } -var testConfigRegistry map[string]*TestConfig - -func LoadTestConfig(coin string) (*TestConfig, error) { - if testConfigRegistry == nil { - b, err := readDataFile(".", "bchain/tests/rpc", "config.json") - if err != nil { - return nil, err - } - var v map[string]*TestConfig - err = json.Unmarshal(b, &v) - if err != nil { - return nil, err - } - testConfigRegistry = v - } - c, found := testConfigRegistry[coin] - if !found { - return nil, errors.New("Test config not found") - } - return c, nil -} - -func LoadRPCConfig(coin string) (json.RawMessage, error) { - t := `{ - "coin_name": "%s", - "rpc_url": "%s", - "rpc_user": "%s", - "rpc_pass": "%s", - "rpc_timeout": 25, - "parse": true - }` - - c, err := LoadTestConfig(coin) - if err != nil { - return json.RawMessage{}, err - } - - return json.RawMessage(fmt.Sprintf(t, coin, c.URL, c.User, c.Pass)), nil -} - func LoadTestData(coin string, parser bchain.BlockChainParser) (*TestData, error) { b, err := readDataFile(".", "bchain/tests/rpc/testdata", coin+".json") if err != nil { diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go deleted file mode 100644 index 675e6015..00000000 --- a/bchain/tests/rpc/rpc.go +++ /dev/null @@ -1,455 +0,0 @@ -package rpc - -import ( - "blockbook/bchain" - "encoding/json" - "math/rand" - "net" - "reflect" - "testing" - "time" - - "github.com/deckarep/golang-set" -) - -type TestConfig struct { - URL string `json:"url"` - User string `json:"user"` - Pass string `json:"pass"` -} - -type TestData struct { - BlockHeight uint32 `json:"blockHeight"` - BlockHash string `json:"blockHash"` - BlockTime int64 `json:"blockTime"` - BlockTxs []string `json:"blockTxs"` - TxDetails map[string]*bchain.Tx `json:"txDetails"` -} - -type Test struct { - Client bchain.BlockChain - TestData *TestData - connected bool -} - -type TestChainFactoryFunc func(json.RawMessage) (bchain.BlockChain, error) - -func NewTest(coin string, factory TestChainFactoryFunc) (*Test, error) { - var ( - connected = true - cli bchain.BlockChain - cfg json.RawMessage - td *TestData - err error - ) - - cfg, err = LoadRPCConfig(coin) - if err != nil { - return nil, err - } - - cli, err = factory(cfg) - if err != nil { - if isNetError(err) { - connected = false - } else { - return nil, err - } - } else { - td, err = LoadTestData(coin, cli.GetChainParser()) - if err != nil { - return nil, err - } - - if td.TxDetails != nil { - parser := cli.GetChainParser() - - for _, tx := range td.TxDetails { - err := setTxAddresses(parser, tx) - if err != nil { - return nil, err - } - } - } - - _, err = cli.GetBlockChainInfo() - if err != nil && isNetError(err) { - connected = false - } - } - - return &Test{Client: cli, TestData: td, connected: connected}, nil -} - -func isNetError(err error) bool { - if _, ok := err.(net.Error); ok { - return true - } - return false -} - -func setTxAddresses(parser bchain.BlockChainParser, tx *bchain.Tx) error { - // pack and unpack transaction in order to get addresses decoded - ugly but works - var tmp *bchain.Tx - b, err := parser.PackTx(tx, 0, 0) - if err == nil { - tmp, _, err = parser.UnpackTx(b) - if err == nil { - for i := 0; i < len(tx.Vout); i++ { - tx.Vout[i].ScriptPubKey.Addresses = tmp.Vout[i].ScriptPubKey.Addresses - } - } - } - return err -} - -func (rt *Test) skipUnconnected(t *testing.T) { - if !rt.connected { - t.Skip("Skipping test, not connected to backend service") - } -} - -func (rt *Test) TestGetBlockHash(t *testing.T) { - rt.skipUnconnected(t) - - hash, err := rt.Client.GetBlockHash(rt.TestData.BlockHeight) - if err != nil { - t.Error(err) - return - } - - if hash != rt.TestData.BlockHash { - t.Errorf("GetBlockHash() got %q, want %q", hash, rt.TestData.BlockHash) - } -} - -func (rt *Test) TestGetBlock(t *testing.T) { - rt.skipUnconnected(t) - - blk, err := rt.Client.GetBlock(rt.TestData.BlockHash, 0) - if err != nil { - t.Error(err) - return - } - - if len(blk.Txs) != len(rt.TestData.BlockTxs) { - t.Errorf("GetBlock() number of transactions: got %d, want %d", len(blk.Txs), len(rt.TestData.BlockTxs)) - } - - for ti, tx := range blk.Txs { - if tx.Txid != rt.TestData.BlockTxs[ti] { - t.Errorf("GetBlock() transaction %d: got %s, want %s", ti, tx.Txid, rt.TestData.BlockTxs[ti]) - } - } - -} - -func (rt *Test) TestGetTransaction(t *testing.T) { - rt.skipUnconnected(t) - - for txid, want := range rt.TestData.TxDetails { - got, err := rt.Client.GetTransaction(txid) - if err != nil { - t.Error(err) - return - } - // Confirmations is variable field, we just check if is set and reset it - if got.Confirmations <= 0 { - t.Errorf("GetTransaction() got struct with invalid Confirmations field") - continue - } - got.Confirmations = 0 - - if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransaction() got %+v, want %+v", got, want) - } - } -} - -func (rt *Test) TestGetTransactionForMempool(t *testing.T) { - rt.skipUnconnected(t) - - for txid, want := range rt.TestData.TxDetails { - // reset fields that are not parsed by BlockChainParser - want.Confirmations, want.Blocktime, want.Time = 0, 0, 0 - - got, err := rt.Client.GetTransactionForMempool(txid) - if err != nil { - t.Fatal(err) - } - // transactions parsed from JSON may contain additional data - got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 - if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransactionForMempool() got %+v, want %+v", got, want) - } - } -} - -func (rt *Test) getMempool(t *testing.T) []string { - txs, err := rt.Client.GetMempool() - if err != nil { - t.Fatal(err) - } - if len(txs) == 0 { - t.Skip("Skipping test, mempool is empty") - } - - return txs -} - -func (rt *Test) getMempoolAddresses(t *testing.T, txs []string) map[string][]string { - txid2addrs := map[string][]string{} - for i := 0; i < len(txs); i++ { - tx, err := rt.Client.GetTransactionForMempool(txs[i]) - if err != nil { - t.Fatal(err) - } - addrs := []string{} - for _, vin := range tx.Vin { - for _, a := range vin.Addresses { - addrs = append(addrs, a) - } - } - for _, vout := range tx.Vout { - for _, a := range vout.ScriptPubKey.Addresses { - addrs = append(addrs, a) - } - } - if len(addrs) > 0 { - txid2addrs[tx.Txid] = addrs - } - } - return txid2addrs -} - -func (rt *Test) TestMempoolSync(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - txs := rt.getMempool(t) - - n, err := rt.Client.ResyncMempool(nil) - if err != nil { - t.Fatal(err) - } - if n == 0 { - // no transactions to test - continue - } - - txs = intersect(txs, rt.getMempool(t)) - if len(txs) == 0 { - // no transactions to test - continue - } - - txid2addrs := rt.getMempoolAddresses(t, txs) - if len(txid2addrs) == 0 { - t.Skip("Skipping test, no addresses in mempool") - } - - for txid, addrs := range txid2addrs { - for _, a := range addrs { - got, err := rt.Client.GetMempoolTransactions(a) - if err != nil { - t.Fatal(err) - } - if !containsString(got, txid) { - t.Errorf("ResyncMempool() - for address %s, transaction %s wasn't found in mempool", a, txid) - return - } - } - } - - // done - return - } - t.Skip("Skipping test, all attempts to sync mempool failed due to network state changes") -} - -func intersect(a, b []string) []string { - setA := mapset.NewSet() - for _, v := range a { - setA.Add(v) - } - setB := mapset.NewSet() - for _, v := range b { - setB.Add(v) - } - inter := setA.Intersect(setB) - res := make([]string, 0, inter.Cardinality()) - for v := range inter.Iter() { - res = append(res, v.(string)) - } - return res -} - -func containsString(slice []string, s string) bool { - for i := 0; i < len(slice); i++ { - if slice[i] == s { - return true - } - } - return false -} - -func (rt *Test) TestGetMempoolEntry(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - txs := rt.getMempool(t) - h, err := rt.Client.GetBestBlockHeight() - if err != nil { - t.Fatal(err) - } - - txid := txs[rand.Intn(len(txs))] - tx, err := rt.Client.GetTransactionForMempool(txid) - if err != nil { - t.Fatal(err) - } - if tx.Confirmations > 0 { - // tx confirmed - continue - } - - e, err := rt.Client.GetMempoolEntry(txid) - if err != nil { - if err, ok := err.(*bchain.RPCError); ok && err.Code == -5 { - // tx confirmed - continue - } - t.Fatal(err) - } - - if d := int(e.Height) - int(h); d < -1 || d > 1 { - t.Errorf("GetMempoolEntry() got height %d, want %d", e.Height, h) - } - if e.Size <= 0 { - t.Errorf("GetMempoolEntry() got zero or negative size %d", e.Size) - } - if e.FeeSat.Sign() != 1 { - t.Errorf("GetMempoolEntry() got zero or negative fee %v", e.FeeSat.String()) - } - - // done - return - } - t.Skip("Skipping test, all attempts to get mempool entry failed due to network state changes") -} - -func (rt *Test) TestEstimateSmartFee(t *testing.T) { - rt.skipUnconnected(t) - - for _, blocks := range []int{1, 2, 3, 5, 10} { - fee, err := rt.Client.EstimateSmartFee(blocks, true) - if err != nil { - t.Error(err) - } - if fee.Sign() == -1 { - sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) - if sf != "-1" { - t.Errorf("EstimateSmartFee() returned unexpected fee rate: %v", sf) - } - } - } -} - -func (rt *Test) TestEstimateFee(t *testing.T) { - rt.skipUnconnected(t) - - for _, blocks := range []int{1, 2, 3, 5, 10} { - fee, err := rt.Client.EstimateFee(blocks) - if err != nil { - t.Error(err) - } - if fee.Sign() == -1 { - sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) - if sf != "-1" { - t.Errorf("EstimateFee() returned unexpected fee rate: %v", sf) - } - } - } -} - -func (rt *Test) TestGetBestBlockHash(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - hash, err := rt.Client.GetBestBlockHash() - if err != nil { - t.Fatal(err) - } - - height, err := rt.Client.GetBestBlockHeight() - if err != nil { - t.Fatal(err) - } - hh, err := rt.Client.GetBlockHash(height) - if err != nil { - t.Fatal(err) - } - if hash != hh { - time.Sleep(time.Millisecond * 100) - continue - } - - // we expect no next block - _, err = rt.Client.GetBlock("", height+1) - if err != nil { - if err != bchain.ErrBlockNotFound { - t.Error(err) - } - return - } - } - t.Error("GetBestBlockHash() didn't get the best hash") -} - -func (rt *Test) TestGetBestBlockHeight(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - height, err := rt.Client.GetBestBlockHeight() - if err != nil { - t.Fatal(err) - } - - // we expect no next block - _, err = rt.Client.GetBlock("", height+1) - if err != nil { - if err != bchain.ErrBlockNotFound { - t.Error(err) - } - return - } - } - t.Error("GetBestBlockHeigh() didn't get the the best heigh") -} - -func (rt *Test) TestGetBlockHeader(t *testing.T) { - rt.skipUnconnected(t) - - want := &bchain.BlockHeader{ - Hash: rt.TestData.BlockHash, - Height: rt.TestData.BlockHeight, - Time: rt.TestData.BlockTime, - } - - got, err := rt.Client.GetBlockHeader(rt.TestData.BlockHash) - if err != nil { - t.Fatal(err) - } - - // Confirmations is variable field, we just check if is set and reset it - if got.Confirmations <= 0 { - t.Fatalf("GetBlockHeader() got struct with invalid Confirmations field") - } - got.Confirmations = 0 - - got.Prev, got.Next = "", "" - - if !reflect.DeepEqual(got, want) { - t.Errorf("GetBlockHeader() got=%+v, want=%+v", got, want) - } -} diff --git a/bchain/tests/rpc/rpc_test.go b/bchain/tests/rpc/rpc_test.go new file mode 100644 index 00000000..8c9f91b5 --- /dev/null +++ b/bchain/tests/rpc/rpc_test.go @@ -0,0 +1,484 @@ +// +build integration + +package rpc + +import ( + "blockbook/bchain" + "blockbook/bchain/coins" + "blockbook/build/tools" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/deckarep/golang-set" +) + +var testMap = map[string]func(t *testing.T, th *TestHandler){ + "GetBlockHash": testGetBlockHash, + "GetBlock": testGetBlock, + "GetTransaction": testGetTransaction, + "GetTransactionForMempool": testGetTransactionForMempool, + "MempoolSync": testMempoolSync, + "EstimateSmartFee": testEstimateSmartFee, + "EstimateFee": testEstimateFee, + "GetBestBlockHash": testGetBestBlockHash, + "GetBestBlockHeight": testGetBestBlockHeight, + "GetBlockHeader": testGetBlockHeader, +} + +type TestHandler struct { + Client bchain.BlockChain + TestData *TestData + connected bool +} + +var notConnectedError = errors.New("Not connected to backend server") + +func TestRPCIntegration(t *testing.T) { + src := os.Getenv("BLOCKBOOK_SRC") + if src == "" { + t.Fatalf("Missing environment variable BLOCKBOOK_SRC") + } + + configsDir := filepath.Join(src, "configs") + templateDir := filepath.Join(src, "build/templates") + + noTests := 0 + skippedTests := make([]string, 0, 10) + + err := filepath.Walk(filepath.Join(configsDir, "coins"), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Name()[0] == '.' { + return nil + } + + n := strings.TrimSuffix(info.Name(), ".json") + c, err := build.LoadConfig(configsDir, n) + if err != nil { + t.Errorf("%s: cannot load configuration: %s", n, err) + return nil + } + if len(c.IntegrationTests["rpc"]) == 0 { + return nil + } + + cfg, err := makeBlockChainConfig(c, templateDir) + if err != nil { + t.Errorf("%s: cannot make blockchain config: %s", n, err) + return nil + } + + t.Run(c.Coin.Alias, func(t *testing.T) { + noTests += 1 + err := runTests(t, c.Coin.Name, c.Coin.Alias, cfg, c.IntegrationTests["rpc"]) + if err != nil { + if err == notConnectedError { + skippedTests = append(skippedTests, c.Coin.Alias) + t.Skip(err) + } + t.Fatal(err) + } + }) + + return nil + }) + + if err != nil { + t.Fatal(err) + } + + if len(skippedTests) > 0 { + t.Errorf("Too many skipped tests: %q", skippedTests) + } +} + +func makeBlockChainConfig(c *build.Config, templateDir string) (json.RawMessage, error) { + outputDir, err := ioutil.TempDir("", "rpc_test") + if err != nil { + return nil, err + } + defer os.RemoveAll(outputDir) + + err = build.GeneratePackageDefinitions(c, templateDir, outputDir) + if err != nil { + return nil, err + } + + b, err := ioutil.ReadFile(filepath.Join(outputDir, "blockbook", "blockchaincfg.json")) + if err != nil { + return nil, err + } + + var v json.RawMessage + err = json.Unmarshal(b, &v) + if err != nil { + return nil, err + } + + return v, nil +} + +func runTests(t *testing.T, coinName, coinAlias string, cfg json.RawMessage, tests []string) error { + cli, err := initBlockChain(coinName, cfg) + if err != nil { + if err == notConnectedError { + return err + } + t.Fatal(err) + } + td, err := LoadTestData(coinAlias, cli.GetChainParser()) + if err != nil { + t.Fatalf("Test data loading failed: %s", err) + } + + if td.TxDetails != nil { + parser := cli.GetChainParser() + + for _, tx := range td.TxDetails { + err := setTxAddresses(parser, tx) + if err != nil { + t.Fatalf("Test data loading failed: %s", err) + } + } + } + + h := TestHandler{Client: cli, TestData: td} + + for _, test := range tests { + if f, found := testMap[test]; found { + t.Run(test, func(t *testing.T) { f(t, &h) }) + } else { + t.Errorf("%s: test not found", test) + continue + } + } + + return nil +} + +func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, error) { + factory, found := coins.BlockChainFactories[coinName] + if !found { + return nil, fmt.Errorf("Factory function not found") + } + + cli, err := factory(cfg, func(_ bchain.NotificationType) {}) + if err != nil { + if isNetError(err) { + return nil, notConnectedError + } + return nil, fmt.Errorf("Factory function failed: %s", err) + } + + err = cli.Initialize() + if err != nil { + if isNetError(err) { + return nil, notConnectedError + } + return nil, fmt.Errorf("BlockChain initialization failed: %s", err) + } + + return cli, nil +} + +func isNetError(err error) bool { + if _, ok := err.(net.Error); ok { + return true + } + return false +} + +func setTxAddresses(parser bchain.BlockChainParser, tx *bchain.Tx) error { + // pack and unpack transaction in order to get addresses decoded - ugly but works + var tmp *bchain.Tx + b, err := parser.PackTx(tx, 0, 0) + if err == nil { + tmp, _, err = parser.UnpackTx(b) + if err == nil { + for i := 0; i < len(tx.Vout); i++ { + tx.Vout[i].ScriptPubKey.Addresses = tmp.Vout[i].ScriptPubKey.Addresses + } + } + } + return err +} + +func testGetBlockHash(t *testing.T, h *TestHandler) { + hash, err := h.Client.GetBlockHash(h.TestData.BlockHeight) + if err != nil { + t.Error(err) + return + } + + if hash != h.TestData.BlockHash { + t.Errorf("GetBlockHash() got %q, want %q", hash, h.TestData.BlockHash) + } +} +func testGetBlock(t *testing.T, h *TestHandler) { + blk, err := h.Client.GetBlock(h.TestData.BlockHash, 0) + if err != nil { + t.Error(err) + return + } + + if len(blk.Txs) != len(h.TestData.BlockTxs) { + t.Errorf("GetBlock() number of transactions: got %d, want %d", len(blk.Txs), len(h.TestData.BlockTxs)) + } + + for ti, tx := range blk.Txs { + if tx.Txid != h.TestData.BlockTxs[ti] { + t.Errorf("GetBlock() transaction %d: got %s, want %s", ti, tx.Txid, h.TestData.BlockTxs[ti]) + } + } +} +func testGetTransaction(t *testing.T, h *TestHandler) { + for txid, want := range h.TestData.TxDetails { + got, err := h.Client.GetTransaction(txid) + if err != nil { + t.Error(err) + return + } + // Confirmations is variable field, we just check if is set and reset it + if got.Confirmations <= 0 { + t.Errorf("GetTransaction() got struct with invalid Confirmations field") + continue + } + got.Confirmations = 0 + + if !reflect.DeepEqual(got, want) { + t.Errorf("GetTransaction() got %+v, want %+v", got, want) + } + } +} +func testGetTransactionForMempool(t *testing.T, h *TestHandler) { + for txid, want := range h.TestData.TxDetails { + // reset fields that are not parsed by BlockChainParser + want.Confirmations, want.Blocktime, want.Time = 0, 0, 0 + + got, err := h.Client.GetTransactionForMempool(txid) + if err != nil { + t.Fatal(err) + } + // transactions parsed from JSON may contain additional data + got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 + if !reflect.DeepEqual(got, want) { + t.Errorf("GetTransactionForMempool() got %+v, want %+v", got, want) + } + } +} +func testMempoolSync(t *testing.T, h *TestHandler) { + for i := 0; i < 3; i++ { + txs := getMempool(t, h) + + n, err := h.Client.ResyncMempool(nil) + if err != nil { + t.Fatal(err) + } + if n == 0 { + // no transactions to test + continue + } + + txs = intersect(txs, getMempool(t, h)) + if len(txs) == 0 { + // no transactions to test + continue + } + + txid2addrs := getMempoolAddresses(t, h, txs) + if len(txid2addrs) == 0 { + t.Skip("Skipping test, no addresses in mempool") + } + + for txid, addrs := range txid2addrs { + for _, a := range addrs { + got, err := h.Client.GetMempoolTransactions(a) + if err != nil { + t.Fatal(err) + } + if !containsString(got, txid) { + t.Errorf("ResyncMempool() - for address %s, transaction %s wasn't found in mempool", a, txid) + return + } + } + } + + // done + return + } + t.Skip("Skipping test, all attempts to sync mempool failed due to network state changes") +} +func testEstimateSmartFee(t *testing.T, h *TestHandler) { + for _, blocks := range []int{1, 2, 3, 5, 10} { + fee, err := h.Client.EstimateSmartFee(blocks, true) + if err != nil { + t.Error(err) + } + if fee.Sign() == -1 { + sf := h.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateSmartFee() returned unexpected fee rate: %v", sf) + } + } + } +} +func testEstimateFee(t *testing.T, h *TestHandler) { + for _, blocks := range []int{1, 2, 3, 5, 10} { + fee, err := h.Client.EstimateFee(blocks) + if err != nil { + t.Error(err) + } + if fee.Sign() == -1 { + sf := h.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateFee() returned unexpected fee rate: %v", sf) + } + } + } +} +func testGetBestBlockHash(t *testing.T, h *TestHandler) { + for i := 0; i < 3; i++ { + hash, err := h.Client.GetBestBlockHash() + if err != nil { + t.Fatal(err) + } + + height, err := h.Client.GetBestBlockHeight() + if err != nil { + t.Fatal(err) + } + hh, err := h.Client.GetBlockHash(height) + if err != nil { + t.Fatal(err) + } + if hash != hh { + time.Sleep(time.Millisecond * 100) + continue + } + + // we expect no next block + _, err = h.Client.GetBlock("", height+1) + if err != nil { + if err != bchain.ErrBlockNotFound { + t.Error(err) + } + return + } + } + t.Error("GetBestBlockHash() didn't get the best hash") +} +func testGetBestBlockHeight(t *testing.T, h *TestHandler) { + for i := 0; i < 3; i++ { + height, err := h.Client.GetBestBlockHeight() + if err != nil { + t.Fatal(err) + } + + // we expect no next block + _, err = h.Client.GetBlock("", height+1) + if err != nil { + if err != bchain.ErrBlockNotFound { + t.Error(err) + } + return + } + } + t.Error("GetBestBlockHeigh() didn't get the the best heigh") +} +func testGetBlockHeader(t *testing.T, h *TestHandler) { + want := &bchain.BlockHeader{ + Hash: h.TestData.BlockHash, + Height: h.TestData.BlockHeight, + Time: h.TestData.BlockTime, + } + + got, err := h.Client.GetBlockHeader(h.TestData.BlockHash) + if err != nil { + t.Fatal(err) + } + + // Confirmations is variable field, we just check if is set and reset it + if got.Confirmations <= 0 { + t.Fatalf("GetBlockHeader() got struct with invalid Confirmations field") + } + got.Confirmations = 0 + + got.Prev, got.Next = "", "" + + if !reflect.DeepEqual(got, want) { + t.Errorf("GetBlockHeader() got=%+v, want=%+v", got, want) + } +} + +func getMempool(t *testing.T, h *TestHandler) []string { + txs, err := h.Client.GetMempool() + if err != nil { + t.Fatal(err) + } + if len(txs) == 0 { + t.Skip("Skipping test, mempool is empty") + } + + return txs +} + +func getMempoolAddresses(t *testing.T, h *TestHandler, txs []string) map[string][]string { + txid2addrs := map[string][]string{} + for i := 0; i < len(txs); i++ { + tx, err := h.Client.GetTransactionForMempool(txs[i]) + if err != nil { + t.Fatal(err) + } + addrs := []string{} + for _, vin := range tx.Vin { + for _, a := range vin.Addresses { + addrs = append(addrs, a) + } + } + for _, vout := range tx.Vout { + for _, a := range vout.ScriptPubKey.Addresses { + addrs = append(addrs, a) + } + } + if len(addrs) > 0 { + txid2addrs[tx.Txid] = addrs + } + } + return txid2addrs +} + +func intersect(a, b []string) []string { + setA := mapset.NewSet() + for _, v := range a { + setA.Add(v) + } + setB := mapset.NewSet() + for _, v := range b { + setB.Add(v) + } + inter := setA.Intersect(setB) + res := make([]string, 0, inter.Cardinality()) + for v := range inter.Iter() { + res = append(res, v.(string)) + } + return res +} + +func containsString(slice []string, s string) bool { + for i := 0; i < len(slice); i++ { + if slice[i] == s { + return true + } + } + return false +} diff --git a/bchain/tests/rpc/testdata/Bcash.json b/bchain/tests/rpc/testdata/bcash.json similarity index 100% rename from bchain/tests/rpc/testdata/Bcash.json rename to bchain/tests/rpc/testdata/bcash.json diff --git a/bchain/tests/rpc/testdata/Bcash_Testnet.json b/bchain/tests/rpc/testdata/bcash_testnet.json similarity index 100% rename from bchain/tests/rpc/testdata/Bcash_Testnet.json rename to bchain/tests/rpc/testdata/bcash_testnet.json diff --git a/bchain/tests/rpc/testdata/Bitcoin.json b/bchain/tests/rpc/testdata/bitcoin.json similarity index 100% rename from bchain/tests/rpc/testdata/Bitcoin.json rename to bchain/tests/rpc/testdata/bitcoin.json diff --git a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json b/bchain/tests/rpc/testdata/bitcoin_testnet.json similarity index 100% rename from bchain/tests/rpc/testdata/Bitcoin_Testnet.json rename to bchain/tests/rpc/testdata/bitcoin_testnet.json diff --git a/bchain/tests/rpc/testdata/Dash.json b/bchain/tests/rpc/testdata/dash.json similarity index 100% rename from bchain/tests/rpc/testdata/Dash.json rename to bchain/tests/rpc/testdata/dash.json diff --git a/bchain/tests/rpc/testdata/Dash_Testnet.json b/bchain/tests/rpc/testdata/dash_testnet.json similarity index 100% rename from bchain/tests/rpc/testdata/Dash_Testnet.json rename to bchain/tests/rpc/testdata/dash_testnet.json diff --git a/bchain/tests/rpc/testdata/Dogecoin.json b/bchain/tests/rpc/testdata/dogecoin.json similarity index 100% rename from bchain/tests/rpc/testdata/Dogecoin.json rename to bchain/tests/rpc/testdata/dogecoin.json diff --git a/bchain/tests/rpc/testdata/Ethereum_Testnet.json b/bchain/tests/rpc/testdata/ethereum_testnet_ropsten.json similarity index 100% rename from bchain/tests/rpc/testdata/Ethereum_Testnet.json rename to bchain/tests/rpc/testdata/ethereum_testnet_ropsten.json diff --git a/bchain/tests/rpc/testdata/Litecoin.json b/bchain/tests/rpc/testdata/litecoin.json similarity index 100% rename from bchain/tests/rpc/testdata/Litecoin.json rename to bchain/tests/rpc/testdata/litecoin.json diff --git a/bchain/tests/rpc/testdata/Monacoin.json b/bchain/tests/rpc/testdata/monacoin.json similarity index 100% rename from bchain/tests/rpc/testdata/Monacoin.json rename to bchain/tests/rpc/testdata/monacoin.json diff --git a/bchain/tests/rpc/testdata/Namecoin.json b/bchain/tests/rpc/testdata/namecoin.json similarity index 100% rename from bchain/tests/rpc/testdata/Namecoin.json rename to bchain/tests/rpc/testdata/namecoin.json diff --git a/bchain/tests/rpc/testdata/Vertcoin.json b/bchain/tests/rpc/testdata/vertcoin.json similarity index 100% rename from bchain/tests/rpc/testdata/Vertcoin.json rename to bchain/tests/rpc/testdata/vertcoin.json diff --git a/bchain/tests/rpc/testdata/Zcash.json b/bchain/tests/rpc/testdata/zcash.json similarity index 100% rename from bchain/tests/rpc/testdata/Zcash.json rename to bchain/tests/rpc/testdata/zcash.json diff --git a/bchain/tests/rpc/testdata/Zcash_Testnet.json b/bchain/tests/rpc/testdata/zcash_testnet.json similarity index 100% rename from bchain/tests/rpc/testdata/Zcash_Testnet.json rename to bchain/tests/rpc/testdata/zcash_testnet.json diff --git a/build/docker/bin/Makefile b/build/docker/bin/Makefile index 2043a202..f972f896 100644 --- a/build/docker/bin/Makefile +++ b/build/docker/bin/Makefile @@ -4,17 +4,20 @@ VERSION ?= devel GITCOMMIT = $(shell cd /src && git describe --tags --always --dirty) BUILDTIME = $(shell date --iso-8601=seconds) LDFLAGS := -X blockbook/common.version=$(VERSION) -X blockbook/common.gitcommit=$(GITCOMMIT) -X blockbook/common.buildtime=$(BUILDTIME) +BLOCKBOOK_SRC := $(GOPATH)/src/blockbook ARGS ?= +export BLOCKBOOK_SRC + all: build tools build: prepare-sources - cd $(GOPATH)/src/blockbook && go build -o $(CURDIR)/blockbook -ldflags="-s -w $(LDFLAGS)" $(ARGS) + cd $(BLOCKBOOK_SRC) && go build -o $(CURDIR)/blockbook -ldflags="-s -w $(LDFLAGS)" $(ARGS) cp $(CURDIR)/blockbook /out/blockbook chown $(PACKAGER) /out/blockbook build-debug: prepare-sources - cd $(GOPATH)/src/blockbook && go build -o $(CURDIR)/blockbook -ldflags="$(LDFLAGS)" $(ARGS) + cd $(BLOCKBOOK_SRC) && go build -o $(CURDIR)/blockbook -ldflags="$(LDFLAGS)" $(ARGS) cp $(CURDIR)/blockbook /out/blockbook chown $(PACKAGER) /out/blockbook @@ -23,20 +26,20 @@ tools: chown $(PACKAGER) /out/{ldb,sst_dump} test: prepare-sources - cd $(GOPATH)/src/blockbook && go test -tags unittest `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) + cd $(BLOCKBOOK_SRC) && go test -tags unittest `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) test-all: prepare-sources - cd $(GOPATH)/src/blockbook && go test -tags 'unittest integration' `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) + cd $(BLOCKBOOK_SRC) && go test -tags 'unittest integration' `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) prepare-sources: @ [ -n "`ls /src 2> /dev/null`" ] || (echo "/src doesn't exist or is empty" 1>&2 && exit 1) - [ -d $(GOPATH)/src/blockbook ] || cp -r /src $(GOPATH)/src/blockbook + [ -d $(BLOCKBOOK_SRC) ] || cp -r /src $(BLOCKBOOK_SRC) $(MAKE) prepare-vendor prepare-vendor: @ if [ "$(UPDATE_VENDOR)" -eq 1 ]; then \ echo "Updating vendor"; \ - rm -rf $(GOPATH)/src/blockbook/vendor && cd $(GOPATH)/src/blockbook && dep ensure -vendor-only ; \ + rm -rf $(BLOCKBOOK_SRC)/vendor && cd $(BLOCKBOOK_SRC) && dep ensure -vendor-only ; \ else \ echo "Update of vendor not demanded, keeping version from src" ; \ fi diff --git a/build/templates/generate.go b/build/templates/generate.go index 20d2f893..6c44ee7b 100644 --- a/build/templates/generate.go +++ b/build/templates/generate.go @@ -1,126 +1,19 @@ package main import ( - "encoding/json" + "blockbook/build/tools" "fmt" - "io" "os" "path/filepath" "strings" - "text/template" - "time" ) const ( - configsDir = "configs" - inputDir = "build/templates" - outputDir = "build/pkg-defs" + configsDir = "configs" + templateDir = "build/templates" + outputDir = "build/pkg-defs" ) -type Config struct { - Meta struct { - BuildDatetime string // generated field - PackageMaintainer string `json:"package_maintainer"` - PackageMaintainerEmail string `json:"package_maintainer_email"` - } - Env struct { - Version string `json:"version"` - BackendInstallPath string `json:"backend_install_path"` - BackendDataPath string `json:"backend_data_path"` - BlockbookInstallPath string `json:"blockbook_install_path"` - BlockbookDataPath string `json:"blockbook_data_path"` - } `json:"env"` - Coin struct { - Name string `json:"name"` - Shortcut string `json:"shortcut"` - Label string `json:"label"` - Alias string `json:"alias"` - } `json:"coin"` - Ports struct { - BackendRPC int `json:"backend_rpc"` - BackendMessageQueue int `json:"backend_message_queue"` - BlockbookInternal int `json:"blockbook_internal"` - BlockbookPublic int `json:"blockbook_public"` - } `json:"ports"` - IPC struct { - RPCURLTemplate string `json:"rpc_url_template"` - RPCUser string `json:"rpc_user"` - RPCPass string `json:"rpc_pass"` - RPCTimeout int `json:"rpc_timeout"` - MessageQueueBindingTemplate string `json:"message_queue_binding_template"` - } `json:"ipc"` - Backend struct { - PackageName string `json:"package_name"` - PackageRevision string `json:"package_revision"` - SystemUser string `json:"system_user"` - Version string `json:"version"` - BinaryURL string `json:"binary_url"` - VerificationType string `json:"verification_type"` - VerificationSource string `json:"verification_source"` - ExtractCommand string `json:"extract_command"` - ExcludeFiles []string `json:"exclude_files"` - ExecCommandTemplate string `json:"exec_command_template"` - LogrotateFilesTemplate string `json:"logrotate_files_template"` - PostinstScriptTemplate string `json:"postinst_script_template"` - ServiceType string `json:"service_type"` - ServiceAdditionalParamsTemplate string `json:"service_additional_params_template"` - ProtectMemory bool `json:"protect_memory"` - Mainnet bool `json:"mainnet"` - ConfigFile string `json:"config_file"` - AdditionalParams interface{} `json:"additional_params"` - } `json:"backend"` - Blockbook struct { - PackageName string `json:"package_name"` - SystemUser string `json:"system_user"` - InternalBindingTemplate string `json:"internal_binding_template"` - PublicBindingTemplate string `json:"public_binding_template"` - ExplorerURL string `json:"explorer_url"` - AdditionalParams string `json:"additional_params"` - BlockChain struct { - Parse bool `json:"parse"` - Subversion string `json:"subversion"` - AddressFormat string `json:"address_format"` - MempoolWorkers int `json:"mempool_workers"` - MempoolSubWorkers int `json:"mempool_sub_workers"` - BlockAddressesToKeep int `json:"block_addresses_to_keep"` - AdditionalParams map[string]json.RawMessage `json:"additional_params"` - } `json:"block_chain"` - } `json:"blockbook"` -} - -func jsonToString(msg json.RawMessage) (string, error) { - d, err := msg.MarshalJSON() - if err != nil { - return "", err - } - return string(d), nil -} - -func (c *Config) ParseTemplate() *template.Template { - templates := map[string]string{ - "IPC.RPCURLTemplate": c.IPC.RPCURLTemplate, - "IPC.MessageQueueBindingTemplate": c.IPC.MessageQueueBindingTemplate, - "Backend.ExecCommandTemplate": c.Backend.ExecCommandTemplate, - "Backend.LogrotateFilesTemplate": c.Backend.LogrotateFilesTemplate, - "Backend.PostinstScriptTemplate": c.Backend.PostinstScriptTemplate, - "Backend.ServiceAdditionalParamsTemplate": c.Backend.ServiceAdditionalParamsTemplate, - "Blockbook.InternalBindingTemplate": c.Blockbook.InternalBindingTemplate, - "Blockbook.PublicBindingTemplate": c.Blockbook.PublicBindingTemplate, - } - - funcMap := template.FuncMap{ - "jsonToString": jsonToString, - } - - t := template.New("").Funcs(funcMap) - - for name, def := range templates { - t = template.Must(t.Parse(fmt.Sprintf(`{{define "%s"}}%s{{end}}`, name, def))) - } - - return t -} - func main() { if len(os.Args) < 2 { var coins []string @@ -136,171 +29,12 @@ func main() { } coin := os.Args[1] - config := loadConfig(coin) - generatePackageDefinitions(config) + config, err := build.LoadConfig(configsDir, coin) + if err == nil { + err = build.GeneratePackageDefinitions(config, templateDir, outputDir) + } + if err != nil { + panic(err) + } fmt.Printf("Package files for %v generated to %v\n", coin, outputDir) } - -func loadConfig(coin string) *Config { - config := new(Config) - - f, err := os.Open(filepath.Join(configsDir, "coins", coin+".json")) - if err != nil { - panic(err) - } - d := json.NewDecoder(f) - err = d.Decode(config) - if err != nil { - panic(err) - } - - f, err = os.Open(filepath.Join(configsDir, "environ.json")) - if err != nil { - panic(err) - } - d = json.NewDecoder(f) - err = d.Decode(&config.Env) - if err != nil { - panic(err) - } - - config.Meta.BuildDatetime = time.Now().Format("Mon, 02 Jan 2006 15:04:05 -0700") - - if !isEmpty(config, "backend") { - switch config.Backend.ServiceType { - case "forking": - case "simple": - default: - panic("Invalid service type: " + config.Backend.ServiceType) - } - - switch config.Backend.VerificationType { - case "": - case "gpg": - case "sha256": - case "gpg-sha256": - default: - panic("Invalid verification type: " + config.Backend.VerificationType) - } - } - - return config -} - -func isEmpty(config *Config, target string) bool { - switch target { - case "backend": - return config.Backend.PackageName == "" - case "blockbook": - return config.Blockbook.PackageName == "" - default: - panic("Invalid target name: " + target) - } -} - -func generatePackageDefinitions(config *Config) { - templ := config.ParseTemplate() - - makeOutputDir(outputDir) - - for _, subdir := range []string{"backend", "blockbook"} { - if isEmpty(config, subdir) { - continue - } - - root := filepath.Join(inputDir, subdir) - - err := os.Mkdir(filepath.Join(outputDir, subdir), 0755) - if err != nil { - panic(err) - } - - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("%s: %s", path, err) - } - - if path == root { - return nil - } - if filepath.Base(path)[0] == '.' { - return nil - } - - subpath := path[len(root)-len(subdir):] - - if info.IsDir() { - err = os.Mkdir(filepath.Join(outputDir, subpath), info.Mode()) - if err != nil { - return fmt.Errorf("%s: %s", path, err) - } - return nil - } - - t := template.Must(templ.Clone()) - t = template.Must(t.ParseFiles(path)) - - err = writeTemplate(filepath.Join(outputDir, subpath), info, t, config) - if err != nil { - return fmt.Errorf("%s: %s", path, err) - } - - return nil - }) - if err != nil { - panic(err) - } - } - - if !isEmpty(config, "backend") { - writeBackendConfigFile(config) - } -} - -func makeOutputDir(path string) { - err := os.RemoveAll(path) - if err == nil { - err = os.Mkdir(path, 0755) - } - if err != nil { - panic(err) - } -} - -func writeTemplate(path string, info os.FileInfo, templ *template.Template, config *Config) error { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, info.Mode()) - if err != nil { - return err - } - defer f.Close() - - err = templ.ExecuteTemplate(f, "main", config) - if err != nil { - return err - } - - return nil -} - -func writeBackendConfigFile(config *Config) { - out, err := os.OpenFile(filepath.Join(outputDir, "backend/backend.conf"), os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - panic(err) - } - defer out.Close() - - if config.Backend.ConfigFile == "" { - return - } else { - in, err := os.Open(filepath.Join(outputDir, "backend/config", config.Backend.ConfigFile)) - if err != nil { - panic(err) - } - defer in.Close() - - _, err = io.Copy(out, in) - if err != nil { - panic(err) - } - } -} diff --git a/build/tools/templates.go b/build/tools/templates.go new file mode 100644 index 00000000..c2e34157 --- /dev/null +++ b/build/tools/templates.go @@ -0,0 +1,288 @@ +package build + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "text/template" + "time" +) + +type Config struct { + Meta struct { + BuildDatetime string // generated field + PackageMaintainer string `json:"package_maintainer"` + PackageMaintainerEmail string `json:"package_maintainer_email"` + } + Env struct { + Version string `json:"version"` + BackendInstallPath string `json:"backend_install_path"` + BackendDataPath string `json:"backend_data_path"` + BlockbookInstallPath string `json:"blockbook_install_path"` + BlockbookDataPath string `json:"blockbook_data_path"` + } `json:"env"` + Coin struct { + Name string `json:"name"` + Shortcut string `json:"shortcut"` + Label string `json:"label"` + Alias string `json:"alias"` + } `json:"coin"` + Ports struct { + BackendRPC int `json:"backend_rpc"` + BackendMessageQueue int `json:"backend_message_queue"` + BlockbookInternal int `json:"blockbook_internal"` + BlockbookPublic int `json:"blockbook_public"` + } `json:"ports"` + IPC struct { + RPCURLTemplate string `json:"rpc_url_template"` + RPCUser string `json:"rpc_user"` + RPCPass string `json:"rpc_pass"` + RPCTimeout int `json:"rpc_timeout"` + MessageQueueBindingTemplate string `json:"message_queue_binding_template"` + } `json:"ipc"` + Backend struct { + PackageName string `json:"package_name"` + PackageRevision string `json:"package_revision"` + SystemUser string `json:"system_user"` + Version string `json:"version"` + BinaryURL string `json:"binary_url"` + VerificationType string `json:"verification_type"` + VerificationSource string `json:"verification_source"` + ExtractCommand string `json:"extract_command"` + ExcludeFiles []string `json:"exclude_files"` + ExecCommandTemplate string `json:"exec_command_template"` + LogrotateFilesTemplate string `json:"logrotate_files_template"` + PostinstScriptTemplate string `json:"postinst_script_template"` + ServiceType string `json:"service_type"` + ServiceAdditionalParamsTemplate string `json:"service_additional_params_template"` + ProtectMemory bool `json:"protect_memory"` + Mainnet bool `json:"mainnet"` + ConfigFile string `json:"config_file"` + AdditionalParams interface{} `json:"additional_params"` + } `json:"backend"` + Blockbook struct { + PackageName string `json:"package_name"` + SystemUser string `json:"system_user"` + InternalBindingTemplate string `json:"internal_binding_template"` + PublicBindingTemplate string `json:"public_binding_template"` + ExplorerURL string `json:"explorer_url"` + AdditionalParams string `json:"additional_params"` + BlockChain struct { + Parse bool `json:"parse"` + Subversion string `json:"subversion"` + AddressFormat string `json:"address_format"` + MempoolWorkers int `json:"mempool_workers"` + MempoolSubWorkers int `json:"mempool_sub_workers"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` + AdditionalParams map[string]json.RawMessage `json:"additional_params"` + } `json:"block_chain"` + } `json:"blockbook"` + IntegrationTests map[string][]string `json:"integration_tests"` +} + +func jsonToString(msg json.RawMessage) (string, error) { + d, err := msg.MarshalJSON() + if err != nil { + return "", err + } + return string(d), nil +} + +func (c *Config) ParseTemplate() *template.Template { + templates := map[string]string{ + "IPC.RPCURLTemplate": c.IPC.RPCURLTemplate, + "IPC.MessageQueueBindingTemplate": c.IPC.MessageQueueBindingTemplate, + "Backend.ExecCommandTemplate": c.Backend.ExecCommandTemplate, + "Backend.LogrotateFilesTemplate": c.Backend.LogrotateFilesTemplate, + "Backend.PostinstScriptTemplate": c.Backend.PostinstScriptTemplate, + "Backend.ServiceAdditionalParamsTemplate": c.Backend.ServiceAdditionalParamsTemplate, + "Blockbook.InternalBindingTemplate": c.Blockbook.InternalBindingTemplate, + "Blockbook.PublicBindingTemplate": c.Blockbook.PublicBindingTemplate, + } + + funcMap := template.FuncMap{ + "jsonToString": jsonToString, + } + + t := template.New("").Funcs(funcMap) + + for name, def := range templates { + t = template.Must(t.Parse(fmt.Sprintf(`{{define "%s"}}%s{{end}}`, name, def))) + } + + return t +} + +func LoadConfig(configsDir, coin string) (*Config, error) { + config := new(Config) + + f, err := os.Open(filepath.Join(configsDir, "coins", coin+".json")) + if err != nil { + return nil, err + } + d := json.NewDecoder(f) + err = d.Decode(config) + if err != nil { + return nil, err + } + + f, err = os.Open(filepath.Join(configsDir, "environ.json")) + if err != nil { + return nil, err + } + d = json.NewDecoder(f) + err = d.Decode(&config.Env) + if err != nil { + return nil, err + } + + config.Meta.BuildDatetime = time.Now().Format("Mon, 02 Jan 2006 15:04:05 -0700") + + if !isEmpty(config, "backend") { + switch config.Backend.ServiceType { + case "forking": + case "simple": + default: + return nil, fmt.Errorf("Invalid service type: %s", config.Backend.ServiceType) + } + + switch config.Backend.VerificationType { + case "": + case "gpg": + case "sha256": + case "gpg-sha256": + default: + return nil, fmt.Errorf("Invalid verification type: %s", config.Backend.VerificationType) + } + } + + return config, nil +} + +func isEmpty(config *Config, target string) bool { + switch target { + case "backend": + return config.Backend.PackageName == "" + case "blockbook": + return config.Blockbook.PackageName == "" + default: + panic("Invalid target name: " + target) + } +} + +func GeneratePackageDefinitions(config *Config, templateDir, outputDir string) error { + templ := config.ParseTemplate() + + err := makeOutputDir(outputDir) + if err != nil { + return err + } + + for _, subdir := range []string{"backend", "blockbook"} { + if isEmpty(config, subdir) { + continue + } + + root := filepath.Join(templateDir, subdir) + + err = os.Mkdir(filepath.Join(outputDir, subdir), 0755) + if err != nil { + return err + } + + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("%s: %s", path, err) + } + + if path == root { + return nil + } + if filepath.Base(path)[0] == '.' { + return nil + } + + subpath := path[len(root)-len(subdir):] + + if info.IsDir() { + err = os.Mkdir(filepath.Join(outputDir, subpath), info.Mode()) + if err != nil { + return fmt.Errorf("%s: %s", path, err) + } + return nil + } + + t := template.Must(templ.Clone()) + t = template.Must(t.ParseFiles(path)) + + err = writeTemplate(filepath.Join(outputDir, subpath), info, t, config) + if err != nil { + return fmt.Errorf("%s: %s", path, err) + } + + return nil + }) + if err != nil { + return err + } + } + + if !isEmpty(config, "backend") { + err = writeBackendConfigFile(config, outputDir) + if err != nil { + return err + } + } + + return nil +} + +func makeOutputDir(path string) error { + err := os.RemoveAll(path) + if err == nil { + err = os.Mkdir(path, 0755) + } + return err +} + +func writeTemplate(path string, info os.FileInfo, templ *template.Template, config *Config) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + defer f.Close() + + err = templ.ExecuteTemplate(f, "main", config) + if err != nil { + return err + } + + return nil +} + +func writeBackendConfigFile(config *Config, outputDir string) error { + out, err := os.OpenFile(filepath.Join(outputDir, "backend/backend.conf"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer out.Close() + + if config.Backend.ConfigFile == "" { + return nil + } else { + in, err := os.Open(filepath.Join(outputDir, "backend/config", config.Backend.ConfigFile)) + if err != nil { + return err + } + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + } + + return nil +} diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 78c74336..69035ab0 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -59,5 +59,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index af58f66a..33427f04 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -59,5 +59,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index ec93e446..76ee0ead 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index 760163be..7d76c723 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/dash.json b/configs/coins/dash.json index d2895671..9da990c2 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -61,5 +61,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index 38407e37..da73d850 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -61,5 +61,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/dogecoin.json b/configs/coins/dogecoin.json index cb9b7a68..a04bda04 100644 --- a/configs/coins/dogecoin.json +++ b/configs/coins/dogecoin.json @@ -63,5 +63,8 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync"] } } diff --git a/configs/coins/ethereum_testnet_ropsten.json b/configs/coins/ethereum_testnet_ropsten.json index a236e16d..6d567f80 100644 --- a/configs/coins/ethereum_testnet_ropsten.json +++ b/configs/coins/ethereum_testnet_ropsten.json @@ -53,5 +53,8 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/litecoin.json b/configs/coins/litecoin.json index e4541bf9..8bd0e15e 100644 --- a/configs/coins/litecoin.json +++ b/configs/coins/litecoin.json @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee"] } } diff --git a/configs/coins/monacoin.json b/configs/coins/monacoin.json index 563b1db6..1a1040a7 100644 --- a/configs/coins/monacoin.json +++ b/configs/coins/monacoin.json @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "wakiyamap", "package_maintainer_email": "wakiyamap@gmail.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee"] } } diff --git a/configs/coins/namecoin.json b/configs/coins/namecoin.json index 754ee628..5514625d 100644 --- a/configs/coins/namecoin.json +++ b/configs/coins/namecoin.json @@ -67,5 +67,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/vertcoin.json b/configs/coins/vertcoin.json index d68a0a0f..5669577a 100644 --- a/configs/coins/vertcoin.json +++ b/configs/coins/vertcoin.json @@ -58,5 +58,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index ab65b246..539029ac 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index c2793c96..c34270ea 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } From 401304d4e8a5d16447542024fbc21d99745242a6 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 11 Sep 2018 17:20:35 +0200 Subject: [PATCH 065/123] Bump Zcash backend version to 2.0.0 --- configs/coins/zcash_testnet.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index 70a3dbd2..43774a1e 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash-testnet", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "1.1.1", - "binary_url": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz", + "version": "2.0.0", + "binary_url": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz", "verification_type": "gpg", - "verification_source": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz.asc", + "verification_source": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", From f6fffefec941951932f693b6adeae892b12d5739 Mon Sep 17 00:00:00 2001 From: Jakub Matys Date: Wed, 12 Sep 2018 10:29:16 +0200 Subject: [PATCH 066/123] Tests fixed --- bchain/tests/rpc/rpc_test.go | 14 +++++++++++--- configs/coins/bcash_testnet.json | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bchain/tests/rpc/rpc_test.go b/bchain/tests/rpc/rpc_test.go index 8c9f91b5..b5658846 100644 --- a/bchain/tests/rpc/rpc_test.go +++ b/bchain/tests/rpc/rpc_test.go @@ -98,7 +98,7 @@ func TestRPCIntegration(t *testing.T) { } if len(skippedTests) > 0 { - t.Errorf("Too many skipped tests: %q", skippedTests) + t.Errorf("Too many skipped tests due to connection issues: %q", skippedTests) } } @@ -442,12 +442,16 @@ func getMempoolAddresses(t *testing.T, h *TestHandler, txs []string) map[string] addrs := []string{} for _, vin := range tx.Vin { for _, a := range vin.Addresses { - addrs = append(addrs, a) + if isSearchableAddr(a) { + addrs = append(addrs, a) + } } } for _, vout := range tx.Vout { for _, a := range vout.ScriptPubKey.Addresses { - addrs = append(addrs, a) + if isSearchableAddr(a) { + addrs = append(addrs, a) + } } } if len(addrs) > 0 { @@ -457,6 +461,10 @@ func getMempoolAddresses(t *testing.T, h *TestHandler, txs []string) map[string] return txid2addrs } +func isSearchableAddr(addr string) bool { + return len(addr) > 3 && addr[:3] != "OP_" +} + func intersect(a, b []string) []string { setA := mapset.NewSet() for _, v := range a { diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index 33427f04..462430ea 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -49,7 +49,7 @@ "block_chain": { "parse": true, "subversion": "/Bitcoin ABC:0.17.1/", - "address_format": "legacy", + "address_format": "cashaddr", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, From 0bfa1fa1b98c6df89c1811fd209746bf36a7565c Mon Sep 17 00:00:00 2001 From: Jakub Matys Date: Wed, 12 Sep 2018 14:15:51 +0200 Subject: [PATCH 067/123] Bump Zcash backend (mainnet) version to 2.0.0 --- configs/coins/zcash.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index 1d697582..a95543f8 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "1.1.1", - "binary_url": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz", + "version": "2.0.0", + "binary_url": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz", "verification_type": "gpg", - "verification_source": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz.asc", + "verification_source": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", From d5e9bb37c9c1e0a74a95e421e0799c7d9e9e0548 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 13 Sep 2018 12:22:33 +0200 Subject: [PATCH 068/123] Bump blockbook version to 0.1.0 --- configs/environ.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/environ.json b/configs/environ.json index 1f293514..3adf012c 100644 --- a/configs/environ.json +++ b/configs/environ.json @@ -1,5 +1,5 @@ { - "version": "0.0.6", + "version": "0.1.0", "backend_install_path": "/opt/coins/nodes", "backend_data_path": "/opt/coins/data", "blockbook_install_path": "/opt/coins/blockbook", From a259d4713cefe5f51156fabb30bdc1555498c0e8 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 13 Sep 2018 13:16:44 +0200 Subject: [PATCH 069/123] Show tx without inputs or outputs in explorer --- static/templates/txdetail.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 9fafc7ec..4ca2a51c 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -27,6 +27,8 @@ {{- end -}} + {{- else -}} + No Inputs {{- end -}} @@ -58,6 +60,8 @@ + {{- else -}} + No Outputs {{- end -}} From c6462bb0e7fe61b21a1e35793f420cd9db31b065 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 13 Sep 2018 14:25:24 +0200 Subject: [PATCH 070/123] Add search for tx/address to explorer --- server/public.go | 51 ++++++++++++++++++++++++++++------ static/css/main.css | 35 +++++++++++++++++++++++ static/templates/base.html | 5 ++++ static/templates/txdetail.html | 8 ++++-- 4 files changed, 89 insertions(+), 10 deletions(-) diff --git a/server/public.go b/server/public.go index a9207822..7aae554a 100644 --- a/server/public.go +++ b/server/public.go @@ -87,6 +87,7 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch // explorer serveMux.HandleFunc(path+"explorer/tx/", s.htmlTemplateHandler(s.explorerTx)) serveMux.HandleFunc(path+"explorer/address/", s.htmlTemplateHandler(s.explorerAddress)) + serveMux.HandleFunc(path+"explorer/search/", s.htmlTemplateHandler(s.explorerSearch)) serveMux.Handle(path+"static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))) // API calls serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex)) @@ -319,6 +320,12 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { return td } +func (s *PublicServer) templateForTx(tx *api.Tx) (tpl, *TemplateData, error) { + data := s.newTemplateData() + data.Tx = tx + return txTpl, data, nil +} + func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { @@ -331,9 +338,16 @@ func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { return errorTpl, nil, err } } + return s.templateForTx(tx) +} + +func (s *PublicServer) templateForAddress(address *api.Address) (tpl, *TemplateData, error) { data := s.newTemplateData() - data.Tx = tx - return txTpl, data, nil + data.AddrStr = address.AddrStr + data.Address = address + data.Page = address.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + return addressTpl, data, nil } func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, error) { @@ -349,12 +363,33 @@ func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, err return errorTpl, nil, err } } - data := s.newTemplateData() - data.AddrStr = address.AddrStr - data.Address = address - data.Page = address.Page - data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) - return addressTpl, data, nil + return s.templateForAddress(address) +} + +func (s *PublicServer) explorerSearch(r *http.Request) (tpl, *TemplateData, error) { + q := strings.TrimSpace(r.URL.Query().Get("q")) + if len(q) > 0 { + } + var tx *api.Tx + var address *api.Address + var err error + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + bestheight, _, err := s.db.GetBestBlock() + if err == nil { + tx, err = s.api.GetTransaction(q, bestheight, true) + if err == nil { + return s.templateForTx(tx) + } + } + address, err = s.api.GetAddress(q, 0, txsOnPage, false) + if err == nil { + return s.templateForAddress(address) + } + } + if err == nil { + err = api.NewApiError(fmt.Sprintf("No matching records found for '%v'", q), true) + } + return errorTpl, nil, err } func getPagingRange(page int, total int) ([]int, int, int) { diff --git a/static/css/main.css b/static/css/main.css index f2ad1378..53928a27 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -31,6 +31,23 @@ h3 { height: 16px; } +.navbar-form { + padding-left: 15px; + padding-bottom: 1px; +} + +.navbar-form .form-control { + background-color: gray; + color: #fff; + border-radius: 3px; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border: 0; + -webkit-box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); + -moz-box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); + box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); +} + @media (min-width: 768px) { .container { max-width: 750px; @@ -47,6 +64,9 @@ h3 { .octicon { height: 24px; } + .navbar-form .form-control { + width: 350px; + } } @media (min-width: 1200px) { @@ -59,6 +79,9 @@ h3 { .octicon { height: 32px; } + .navbar-form .form-control { + width: 360px; + } } #header { @@ -194,6 +217,18 @@ h3 { padding-left: .25rem; } +::-webkit-input-placeholder { + color: #CCC!important; + font-style: italic; + font-size: 14px; +} + +::-moz-placeholder { + color: #CCC!important; + font-style: italic; + font-size: 14px; +} + .h-container { display: -webkit-box; display: -ms-flexbox; diff --git a/static/templates/base.html b/static/templates/base.html index 5ab2fb3e..2915944c 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -32,6 +32,11 @@ {{.CoinName}} Explorer + + +
  • TREZOR diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 4ca2a51c..fbef2681 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -28,7 +28,9 @@ {{- else -}} - No Inputs + + No Inputs + {{- end -}} @@ -61,7 +63,9 @@ {{- else -}} - No Outputs + + No Outputs + {{- end -}} From 955dcfa34656dbf870855a177ecf0b6103a9818c Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 13 Sep 2018 22:59:24 +0200 Subject: [PATCH 071/123] Tune explorer navbar --- static/css/main.css | 12 +++++++++--- static/templates/base.html | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/static/css/main.css b/static/css/main.css index 53928a27..7177a4f9 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -32,7 +32,6 @@ h3 { } .navbar-form { - padding-left: 15px; padding-bottom: 1px; } @@ -65,7 +64,7 @@ h3 { height: 24px; } .navbar-form .form-control { - width: 350px; + width: 230px; } } @@ -106,7 +105,6 @@ h3 { color: rgba(255, 255, 255); font-weight: bold; font-size: 19px; - padding-left: 35px; } .trezor-logo-svg-white svg { @@ -161,6 +159,10 @@ h3 { border-radius: .25em; } +.txvalues:not(:last-child) { + margin-right: 5px; +} + .txvalues-default { background-color: #EBEBEB; color: #333; @@ -217,6 +219,10 @@ h3 { padding-left: .25rem; } +.navbar-text .nav-link { + padding: 0; +} + ::-webkit-input-placeholder { color: #CCC!important; font-style: italic; diff --git a/static/templates/base.html b/static/templates/base.html index 2915944c..3a1d3283 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -29,15 +29,23 @@
- - {{.CoinName}} Explorer + + {{.CoinName}} Explorer - + + -
    +
    • TREZOR
      • From 13dfa46edc63b58f9ab054eb53b0b471bb5d084b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 13 Sep 2018 23:52:39 +0200 Subject: [PATCH 072/123] Add QR code to address view in explorer --- static/css/main.css | 6 ++++- static/js/qrcode.min.js | 1 + static/templates/address.html | 51 ++++++++++++++++++++--------------- 3 files changed, 36 insertions(+), 22 deletions(-) create mode 100755 static/js/qrcode.min.js diff --git a/static/css/main.css b/static/css/main.css index 7177a4f9..51edec97 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -189,7 +189,11 @@ h3 { } .data-div { - margin: 20px 0; + margin: 20px 0 30px 0; +} + +.data-div .col-md-10 { + padding-left: 0; } .data-table { diff --git a/static/js/qrcode.min.js b/static/js/qrcode.min.js new file mode 100755 index 00000000..993e88f3 --- /dev/null +++ b/static/js/qrcode.min.js @@ -0,0 +1 @@ +var QRCode;!function(){function a(a){this.mode=c.MODE_8BIT_BYTE,this.data=a,this.parsedData=[];for(var b=[],d=0,e=this.data.length;e>d;d++){var f=this.data.charCodeAt(d);f>65536?(b[0]=240|(1835008&f)>>>18,b[1]=128|(258048&f)>>>12,b[2]=128|(4032&f)>>>6,b[3]=128|63&f):f>2048?(b[0]=224|(61440&f)>>>12,b[1]=128|(4032&f)>>>6,b[2]=128|63&f):f>128?(b[0]=192|(1984&f)>>>6,b[1]=128|63&f):b[0]=f,this.parsedData=this.parsedData.concat(b)}this.parsedData.length!=this.data.length&&(this.parsedData.unshift(191),this.parsedData.unshift(187),this.parsedData.unshift(239))}function b(a,b){this.typeNumber=a,this.errorCorrectLevel=b,this.modules=null,this.moduleCount=0,this.dataCache=null,this.dataList=[]}function i(a,b){if(void 0==a.length)throw new Error(a.length+"/"+b);for(var c=0;c=f;f++){var h=0;switch(b){case d.L:h=l[f][0];break;case d.M:h=l[f][1];break;case d.Q:h=l[f][2];break;case d.H:h=l[f][3]}if(h>=e)break;c++}if(c>l.length)throw new Error("Too long data");return c}function s(a){var b=encodeURI(a).toString().replace(/\%[0-9a-fA-F]{2}/g,"a");return b.length+(b.length!=a?3:0)}a.prototype={getLength:function(){return this.parsedData.length},write:function(a){for(var b=0,c=this.parsedData.length;c>b;b++)a.put(this.parsedData[b],8)}},b.prototype={addData:function(b){var c=new a(b);this.dataList.push(c),this.dataCache=null},isDark:function(a,b){if(0>a||this.moduleCount<=a||0>b||this.moduleCount<=b)throw new Error(a+","+b);return this.modules[a][b]},getModuleCount:function(){return this.moduleCount},make:function(){this.makeImpl(!1,this.getBestMaskPattern())},makeImpl:function(a,c){this.moduleCount=4*this.typeNumber+17,this.modules=new Array(this.moduleCount);for(var d=0;d=7&&this.setupTypeNumber(a),null==this.dataCache&&(this.dataCache=b.createData(this.typeNumber,this.errorCorrectLevel,this.dataList)),this.mapData(this.dataCache,c)},setupPositionProbePattern:function(a,b){for(var c=-1;7>=c;c++)if(!(-1>=a+c||this.moduleCount<=a+c))for(var d=-1;7>=d;d++)-1>=b+d||this.moduleCount<=b+d||(this.modules[a+c][b+d]=c>=0&&6>=c&&(0==d||6==d)||d>=0&&6>=d&&(0==c||6==c)||c>=2&&4>=c&&d>=2&&4>=d?!0:!1)},getBestMaskPattern:function(){for(var a=0,b=0,c=0;8>c;c++){this.makeImpl(!0,c);var d=f.getLostPoint(this);(0==c||a>d)&&(a=d,b=c)}return b},createMovieClip:function(a,b,c){var d=a.createEmptyMovieClip(b,c),e=1;this.make();for(var f=0;f=g;g++)for(var h=-2;2>=h;h++)this.modules[d+g][e+h]=-2==g||2==g||-2==h||2==h||0==g&&0==h?!0:!1}},setupTypeNumber:function(a){for(var b=f.getBCHTypeNumber(this.typeNumber),c=0;18>c;c++){var d=!a&&1==(1&b>>c);this.modules[Math.floor(c/3)][c%3+this.moduleCount-8-3]=d}for(var c=0;18>c;c++){var d=!a&&1==(1&b>>c);this.modules[c%3+this.moduleCount-8-3][Math.floor(c/3)]=d}},setupTypeInfo:function(a,b){for(var c=this.errorCorrectLevel<<3|b,d=f.getBCHTypeInfo(c),e=0;15>e;e++){var g=!a&&1==(1&d>>e);6>e?this.modules[e][8]=g:8>e?this.modules[e+1][8]=g:this.modules[this.moduleCount-15+e][8]=g}for(var e=0;15>e;e++){var g=!a&&1==(1&d>>e);8>e?this.modules[8][this.moduleCount-e-1]=g:9>e?this.modules[8][15-e-1+1]=g:this.modules[8][15-e-1]=g}this.modules[this.moduleCount-8][8]=!a},mapData:function(a,b){for(var c=-1,d=this.moduleCount-1,e=7,g=0,h=this.moduleCount-1;h>0;h-=2)for(6==h&&h--;;){for(var i=0;2>i;i++)if(null==this.modules[d][h-i]){var j=!1;g>>e));var k=f.getMask(b,d,h-i);k&&(j=!j),this.modules[d][h-i]=j,e--,-1==e&&(g++,e=7)}if(d+=c,0>d||this.moduleCount<=d){d-=c,c=-c;break}}}},b.PAD0=236,b.PAD1=17,b.createData=function(a,c,d){for(var e=j.getRSBlocks(a,c),g=new k,h=0;h8*l)throw new Error("code length overflow. ("+g.getLengthInBits()+">"+8*l+")");for(g.getLengthInBits()+4<=8*l&&g.put(0,4);0!=g.getLengthInBits()%8;)g.putBit(!1);for(;;){if(g.getLengthInBits()>=8*l)break;if(g.put(b.PAD0,8),g.getLengthInBits()>=8*l)break;g.put(b.PAD1,8)}return b.createBytes(g,e)},b.createBytes=function(a,b){for(var c=0,d=0,e=0,g=new Array(b.length),h=new Array(b.length),j=0;j=0?p.get(q):0}}for(var r=0,m=0;mm;m++)for(var j=0;jm;m++)for(var j=0;j=0;)b^=f.G15<=0;)b^=f.G18<>>=1;return b},getPatternPosition:function(a){return f.PATTERN_POSITION_TABLE[a-1]},getMask:function(a,b,c){switch(a){case e.PATTERN000:return 0==(b+c)%2;case e.PATTERN001:return 0==b%2;case e.PATTERN010:return 0==c%3;case e.PATTERN011:return 0==(b+c)%3;case e.PATTERN100:return 0==(Math.floor(b/2)+Math.floor(c/3))%2;case e.PATTERN101:return 0==b*c%2+b*c%3;case e.PATTERN110:return 0==(b*c%2+b*c%3)%2;case e.PATTERN111:return 0==(b*c%3+(b+c)%2)%2;default:throw new Error("bad maskPattern:"+a)}},getErrorCorrectPolynomial:function(a){for(var b=new i([1],0),c=0;a>c;c++)b=b.multiply(new i([1,g.gexp(c)],0));return b},getLengthInBits:function(a,b){if(b>=1&&10>b)switch(a){case c.MODE_NUMBER:return 10;case c.MODE_ALPHA_NUM:return 9;case c.MODE_8BIT_BYTE:return 8;case c.MODE_KANJI:return 8;default:throw new Error("mode:"+a)}else if(27>b)switch(a){case c.MODE_NUMBER:return 12;case c.MODE_ALPHA_NUM:return 11;case c.MODE_8BIT_BYTE:return 16;case c.MODE_KANJI:return 10;default:throw new Error("mode:"+a)}else{if(!(41>b))throw new Error("type:"+b);switch(a){case c.MODE_NUMBER:return 14;case c.MODE_ALPHA_NUM:return 13;case c.MODE_8BIT_BYTE:return 16;case c.MODE_KANJI:return 12;default:throw new Error("mode:"+a)}}},getLostPoint:function(a){for(var b=a.getModuleCount(),c=0,d=0;b>d;d++)for(var e=0;b>e;e++){for(var f=0,g=a.isDark(d,e),h=-1;1>=h;h++)if(!(0>d+h||d+h>=b))for(var i=-1;1>=i;i++)0>e+i||e+i>=b||(0!=h||0!=i)&&g==a.isDark(d+h,e+i)&&f++;f>5&&(c+=3+f-5)}for(var d=0;b-1>d;d++)for(var e=0;b-1>e;e++){var j=0;a.isDark(d,e)&&j++,a.isDark(d+1,e)&&j++,a.isDark(d,e+1)&&j++,a.isDark(d+1,e+1)&&j++,(0==j||4==j)&&(c+=3)}for(var d=0;b>d;d++)for(var e=0;b-6>e;e++)a.isDark(d,e)&&!a.isDark(d,e+1)&&a.isDark(d,e+2)&&a.isDark(d,e+3)&&a.isDark(d,e+4)&&!a.isDark(d,e+5)&&a.isDark(d,e+6)&&(c+=40);for(var e=0;b>e;e++)for(var d=0;b-6>d;d++)a.isDark(d,e)&&!a.isDark(d+1,e)&&a.isDark(d+2,e)&&a.isDark(d+3,e)&&a.isDark(d+4,e)&&!a.isDark(d+5,e)&&a.isDark(d+6,e)&&(c+=40);for(var k=0,e=0;b>e;e++)for(var d=0;b>d;d++)a.isDark(d,e)&&k++;var l=Math.abs(100*k/b/b-50)/5;return c+=10*l}},g={glog:function(a){if(1>a)throw new Error("glog("+a+")");return g.LOG_TABLE[a]},gexp:function(a){for(;0>a;)a+=255;for(;a>=256;)a-=255;return g.EXP_TABLE[a]},EXP_TABLE:new Array(256),LOG_TABLE:new Array(256)},h=0;8>h;h++)g.EXP_TABLE[h]=1<h;h++)g.EXP_TABLE[h]=g.EXP_TABLE[h-4]^g.EXP_TABLE[h-5]^g.EXP_TABLE[h-6]^g.EXP_TABLE[h-8];for(var h=0;255>h;h++)g.LOG_TABLE[g.EXP_TABLE[h]]=h;i.prototype={get:function(a){return this.num[a]},getLength:function(){return this.num.length},multiply:function(a){for(var b=new Array(this.getLength()+a.getLength()-1),c=0;cf;f++)for(var g=c[3*f+0],h=c[3*f+1],i=c[3*f+2],k=0;g>k;k++)e.push(new j(h,i));return e},j.getRsBlockTable=function(a,b){switch(b){case d.L:return j.RS_BLOCK_TABLE[4*(a-1)+0];case d.M:return j.RS_BLOCK_TABLE[4*(a-1)+1];case d.Q:return j.RS_BLOCK_TABLE[4*(a-1)+2];case d.H:return j.RS_BLOCK_TABLE[4*(a-1)+3];default:return void 0}},k.prototype={get:function(a){var b=Math.floor(a/8);return 1==(1&this.buffer[b]>>>7-a%8)},put:function(a,b){for(var c=0;b>c;c++)this.putBit(1==(1&a>>>b-c-1))},getLengthInBits:function(){return this.length},putBit:function(a){var b=Math.floor(this.length/8);this.buffer.length<=b&&this.buffer.push(0),a&&(this.buffer[b]|=128>>>this.length%8),this.length++}};var l=[[17,14,11,7],[32,26,20,14],[53,42,32,24],[78,62,46,34],[106,84,60,44],[134,106,74,58],[154,122,86,64],[192,152,108,84],[230,180,130,98],[271,213,151,119],[321,251,177,137],[367,287,203,155],[425,331,241,177],[458,362,258,194],[520,412,292,220],[586,450,322,250],[644,504,364,280],[718,560,394,310],[792,624,442,338],[858,666,482,382],[929,711,509,403],[1003,779,565,439],[1091,857,611,461],[1171,911,661,511],[1273,997,715,535],[1367,1059,751,593],[1465,1125,805,625],[1528,1190,868,658],[1628,1264,908,698],[1732,1370,982,742],[1840,1452,1030,790],[1952,1538,1112,842],[2068,1628,1168,898],[2188,1722,1228,958],[2303,1809,1283,983],[2431,1911,1351,1051],[2563,1989,1423,1093],[2699,2099,1499,1139],[2809,2213,1579,1219],[2953,2331,1663,1273]],o=function(){var a=function(a,b){this._el=a,this._htOption=b};return a.prototype.draw=function(a){function g(a,b){var c=document.createElementNS("http://www.w3.org/2000/svg",a);for(var d in b)b.hasOwnProperty(d)&&c.setAttribute(d,b[d]);return c}var b=this._htOption,c=this._el,d=a.getModuleCount();Math.floor(b.width/d),Math.floor(b.height/d),this.clear();var h=g("svg",{viewBox:"0 0 "+String(d)+" "+String(d),width:"100%",height:"100%",fill:b.colorLight});h.setAttributeNS("http://www.w3.org/2000/xmlns/","xmlns:xlink","http://www.w3.org/1999/xlink"),c.appendChild(h),h.appendChild(g("rect",{fill:b.colorDark,width:"1",height:"1",id:"template"}));for(var i=0;d>i;i++)for(var j=0;d>j;j++)if(a.isDark(i,j)){var k=g("use",{x:String(i),y:String(j)});k.setAttributeNS("http://www.w3.org/1999/xlink","href","#template"),h.appendChild(k)}},a.prototype.clear=function(){for(;this._el.hasChildNodes();)this._el.removeChild(this._el.lastChild)},a}(),p="svg"===document.documentElement.tagName.toLowerCase(),q=p?o:m()?function(){function a(){this._elImage.src=this._elCanvas.toDataURL("image/png"),this._elImage.style.display="block",this._elCanvas.style.display="none"}function d(a,b){var c=this;if(c._fFail=b,c._fSuccess=a,null===c._bSupportDataURI){var d=document.createElement("img"),e=function(){c._bSupportDataURI=!1,c._fFail&&_fFail.call(c)},f=function(){c._bSupportDataURI=!0,c._fSuccess&&c._fSuccess.call(c)};return d.onabort=e,d.onerror=e,d.onload=f,d.src="data:image/gif;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==",void 0}c._bSupportDataURI===!0&&c._fSuccess?c._fSuccess.call(c):c._bSupportDataURI===!1&&c._fFail&&c._fFail.call(c)}if(this._android&&this._android<=2.1){var b=1/window.devicePixelRatio,c=CanvasRenderingContext2D.prototype.drawImage;CanvasRenderingContext2D.prototype.drawImage=function(a,d,e,f,g,h,i,j){if("nodeName"in a&&/img/i.test(a.nodeName))for(var l=arguments.length-1;l>=1;l--)arguments[l]=arguments[l]*b;else"undefined"==typeof j&&(arguments[1]*=b,arguments[2]*=b,arguments[3]*=b,arguments[4]*=b);c.apply(this,arguments)}}var e=function(a,b){this._bIsPainted=!1,this._android=n(),this._htOption=b,this._elCanvas=document.createElement("canvas"),this._elCanvas.width=b.width,this._elCanvas.height=b.height,a.appendChild(this._elCanvas),this._el=a,this._oContext=this._elCanvas.getContext("2d"),this._bIsPainted=!1,this._elImage=document.createElement("img"),this._elImage.style.display="none",this._el.appendChild(this._elImage),this._bSupportDataURI=null};return e.prototype.draw=function(a){var b=this._elImage,c=this._oContext,d=this._htOption,e=a.getModuleCount(),f=d.width/e,g=d.height/e,h=Math.round(f),i=Math.round(g);b.style.display="none",this.clear();for(var j=0;e>j;j++)for(var k=0;e>k;k++){var l=a.isDark(j,k),m=k*f,n=j*g;c.strokeStyle=l?d.colorDark:d.colorLight,c.lineWidth=1,c.fillStyle=l?d.colorDark:d.colorLight,c.fillRect(m,n,f,g),c.strokeRect(Math.floor(m)+.5,Math.floor(n)+.5,h,i),c.strokeRect(Math.ceil(m)-.5,Math.ceil(n)-.5,h,i)}this._bIsPainted=!0},e.prototype.makeImage=function(){this._bIsPainted&&d.call(this,a)},e.prototype.isPainted=function(){return this._bIsPainted},e.prototype.clear=function(){this._oContext.clearRect(0,0,this._elCanvas.width,this._elCanvas.height),this._bIsPainted=!1},e.prototype.round=function(a){return a?Math.floor(1e3*a)/1e3:a},e}():function(){var a=function(a,b){this._el=a,this._htOption=b};return a.prototype.draw=function(a){for(var b=this._htOption,c=this._el,d=a.getModuleCount(),e=Math.floor(b.width/d),f=Math.floor(b.height/d),g=[''],h=0;d>h;h++){g.push("");for(var i=0;d>i;i++)g.push('');g.push("")}g.push("
      "),c.innerHTML=g.join("");var j=c.childNodes[0],k=(b.width-j.offsetWidth)/2,l=(b.height-j.offsetHeight)/2;k>0&&l>0&&(j.style.margin=l+"px "+k+"px")},a.prototype.clear=function(){this._el.innerHTML=""},a}();QRCode=function(a,b){if(this._htOption={width:256,height:256,typeNumber:4,colorDark:"#000000",colorLight:"#ffffff",correctLevel:d.H},"string"==typeof b&&(b={text:b}),b)for(var c in b)this._htOption[c]=b[c];"string"==typeof a&&(a=document.getElementById(a)),this._android=n(),this._el=a,this._oQRCode=null,this._oDrawing=new q(this._el,this._htOption),this._htOption.text&&this.makeCode(this._htOption.text)},QRCode.prototype.makeCode=function(a){this._oQRCode=new b(r(a,this._htOption.correctLevel),this._htOption.correctLevel),this._oQRCode.addData(a),this._oQRCode.make(),this._el.title=a,this._oDrawing.draw(this._oQRCode),this.makeImage()},QRCode.prototype.makeImage=function(){"function"==typeof this._oDrawing.makeImage&&(!this._android||this._android>=3)&&this._oDrawing.makeImage()},QRCode.prototype.clear=function(){this._oDrawing.clear()},QRCode.CorrectLevel=d}(); \ No newline at end of file diff --git a/static/templates/address.html b/static/templates/address.html index b9100cb7..13cf1d0a 100644 --- a/static/templates/address.html +++ b/static/templates/address.html @@ -6,27 +6,36 @@ {{$addr.AddrStr}}

      Confirmed

      -
      - - - - - - - - - - - - - - - - - - - -
      Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}
      Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}
      Final Balance{{formatAmount $addr.Balance}} {{$cs}}
      No. Transactions{{$addr.TxApperances}}
      +
      +
      + + + + + + + + + + + + + + + + + + + +
      Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}
      Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}
      Final Balance{{formatAmount $addr.Balance}} {{$cs}}
      No. Transactions{{$addr.TxApperances}}
      +
      +
      +
      + + +
      {{- if $addr.UnconfirmedTxApperances -}}

      Unconfirmed

      From 435b6f4e802aa38f96ce5bdf9c237c057ff5bdf0 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 14 Sep 2018 00:13:25 +0200 Subject: [PATCH 073/123] Show search results using redirect in explorer --- server/public.go | 72 +++++++++++++++++------------------ static/templates/address.html | 2 +- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/server/public.go b/server/public.go index 7aae554a..403c80e7 100644 --- a/server/public.go +++ b/server/public.go @@ -222,7 +222,7 @@ func (s *PublicServer) newTemplateDataWithError(text string) *TemplateData { Error: &api.ApiError{Text: text}, } } -func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { var t tpl var data *TemplateData @@ -237,9 +237,12 @@ func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, * data = s.newTemplateDataWithError("Internal server error") } } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := s.templates[t].ExecuteTemplate(w, "base.html", data); err != nil { - glog.Error(err) + // noTpl means the handler completely handled the request + if t != noTpl { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := s.templates[t].ExecuteTemplate(w, "base.html", data); err != nil { + glog.Error(err) + } } }() if s.debug { @@ -247,8 +250,8 @@ func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, * // to reflect changes during development s.templates = parseTemplates() } - t, data, err = handler(r) - if err != nil || data == nil { + t, data, err = handler(w, r) + if err != nil || (data == nil && t != noTpl) { t = errorTpl if apiErr, ok := err.(*api.ApiError); ok { data = s.newTemplateData() @@ -270,7 +273,8 @@ func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, * type tpl int const ( - errorTpl = tpl(iota) + noTpl = tpl(iota) + errorTpl txTpl addressTpl @@ -320,13 +324,7 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { return td } -func (s *PublicServer) templateForTx(tx *api.Tx) (tpl, *TemplateData, error) { - data := s.newTemplateData() - data.Tx = tx - return txTpl, data, nil -} - -func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { +func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] @@ -338,19 +336,12 @@ func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { return errorTpl, nil, err } } - return s.templateForTx(tx) -} - -func (s *PublicServer) templateForAddress(address *api.Address) (tpl, *TemplateData, error) { data := s.newTemplateData() - data.AddrStr = address.AddrStr - data.Address = address - data.Page = address.Page - data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) - return addressTpl, data, nil + data.Tx = tx + return txTpl, data, nil } -func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, error) { +func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var address *api.Address var err error if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { @@ -363,27 +354,34 @@ func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, err return errorTpl, nil, err } } - return s.templateForAddress(address) + data := s.newTemplateData() + data.AddrStr = address.AddrStr + data.Address = address + data.Page = address.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + return addressTpl, data, nil } -func (s *PublicServer) explorerSearch(r *http.Request) (tpl, *TemplateData, error) { +func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { q := strings.TrimSpace(r.URL.Query().Get("q")) - if len(q) > 0 { - } var tx *api.Tx var address *api.Address var err error - if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { - bestheight, _, err := s.db.GetBestBlock() - if err == nil { - tx, err = s.api.GetTransaction(q, bestheight, true) + if len(q) > 0 { + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + bestheight, _, err := s.db.GetBestBlock() if err == nil { - return s.templateForTx(tx) + tx, err = s.api.GetTransaction(q, bestheight, false) + if err == nil { + http.Redirect(w, r, joinURL("/explorer/tx/", tx.Txid), 302) + return noTpl, nil, nil + } + } + address, err = s.api.GetAddress(q, 0, 1, true) + if err == nil { + http.Redirect(w, r, joinURL("/explorer/address/", address.AddrStr), 302) + return noTpl, nil, nil } - } - address, err = s.api.GetAddress(q, 0, txsOnPage, false) - if err == nil { - return s.templateForAddress(address) } } if err == nil { diff --git a/static/templates/address.html b/static/templates/address.html index 13cf1d0a..7300d079 100644 --- a/static/templates/address.html +++ b/static/templates/address.html @@ -30,7 +30,7 @@
      -
      +