From b464f282a9203905a34dc7928948470e1f5c0fa8 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 24 Jul 2018 15:58:37 +0200 Subject: [PATCH 01/18] Use big.Int for all amounts --- api/types.go | 59 +++++++++++------------ api/worker.go | 63 ++++++++++++------------ bchain/baseparser.go | 60 +++++++++++++++++++++-- bchain/coins/bch/bcashrpc.go | 14 ++++-- bchain/coins/blockchain.go | 5 +- bchain/coins/btc/bitcoinparser.go | 6 ++- bchain/coins/btc/bitcoinrpc.go | 72 +++++++++++++++++++--------- bchain/coins/dash/dashrpc.go | 6 +-- bchain/coins/dogecoin/dogecoinrpc.go | 6 +-- bchain/coins/eth/ethparser.go | 20 ++++---- bchain/coins/eth/ethrpc.go | 10 ++-- bchain/coins/litecoin/litecoinrpc.go | 6 +-- bchain/coins/namecoin/namecoinrpc.go | 6 +-- bchain/coins/vertcoin/vertcoinrpc.go | 6 +-- bchain/coins/zec/zcashrpc.go | 9 +--- bchain/tx.pb.go | 52 ++++++++++---------- bchain/tx.proto | 2 +- bchain/types.go | 21 +++++--- server/public.go | 6 ++- server/socketio.go | 19 ++++---- 20 files changed, 261 insertions(+), 187 deletions(-) diff --git a/api/types.go b/api/types.go index b605bd30..287e04af 100644 --- a/api/types.go +++ b/api/types.go @@ -1,5 +1,7 @@ package api +import "math/big" + type ScriptSig struct { Hex string `json:"hex"` Asm string `json:"asm,omitempty"` @@ -12,8 +14,8 @@ type Vin struct { N int `json:"n"` ScriptSig ScriptSig `json:"scriptSig"` Addr string `json:"addr"` - ValueSat int64 `json:"valueSat"` - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` } type ScriptPubKey struct { @@ -23,7 +25,8 @@ type ScriptPubKey struct { Type string `json:"type,omitempty"` } type Vout struct { - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` N int `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` SpentTxID string `json:"spentTxId,omitempty"` @@ -32,34 +35,30 @@ type Vout struct { } type Tx struct { - Txid string `json:"txid"` - Version int32 `json:"version,omitempty"` - Locktime uint32 `json:"locktime,omitempty"` - Vin []Vin `json:"vin"` - Vout []Vout `json:"vout"` - Blockhash string `json:"blockhash,omitempty"` - Blockheight int `json:"blockheight"` - Confirmations uint32 `json:"confirmations"` - Time int64 `json:"time,omitempty"` - Blocktime int64 `json:"blocktime"` - ValueOut float64 `json:"valueOut"` - Size int `json:"size,omitempty"` - ValueIn float64 `json:"valueIn"` - Fees float64 `json:"fees"` - WithSpends bool `json:"withSpends,omitempty"` + Txid string `json:"txid"` + Version int32 `json:"version,omitempty"` + Locktime uint32 `json:"locktime,omitempty"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` + Blockhash string `json:"blockhash,omitempty"` + Blockheight int `json:"blockheight"` + Confirmations uint32 `json:"confirmations"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime"` + ValueOut string `json:"valueOut"` + Size int `json:"size,omitempty"` + ValueIn string `json:"valueIn"` + Fees string `json:"fees"` + WithSpends bool `json:"withSpends,omitempty"` } type Address struct { - AddrStr string `json:"addrStr"` - Balance float64 `json:"balance"` - BalanceSat int64 `json:"balanceSat"` - TotalReceived float64 `json:"totalReceived"` - TotalReceivedSat int64 `json:"totalReceivedSat"` - TotalSent float64 `json:"totalSent"` - TotalSentSat int64 `json:"totalSentSat"` - UnconfirmedBalance float64 `json:"unconfirmedBalance"` - UnconfirmedBalanceSat int64 `json:"unconfirmedBalanceSat"` - UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` - TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"transactions"` + AddrStr string `json:"addrStr"` + Balance string `json:"balance"` + TotalReceived string `json:"totalReceived"` + TotalSent string `json:"totalSent"` + UnconfirmedBalance string `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*Tx `json:"transactions"` } diff --git a/api/worker.go b/api/worker.go index 4ad421d3..ea2870e6 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "math/big" "github.com/golang/glog" ) @@ -44,7 +45,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) return nil, err } } - var valIn, valOut, fees float64 + var valInSat, valOutSat, feesSat big.Int vins := make([]Vin, len(bchainTx.Vin)) for i := range bchainTx.Vin { bchainVin := &bchainTx.Vin[i] @@ -61,9 +62,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) } if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] - vin.Value = vout.Value - valIn += vout.Value - vin.ValueSat = int64(vout.Value*1E8 + 0.5) + vin.ValueSat = vout.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) + valInSat.Add(&valInSat, &vout.ValueSat) if vout.Address != nil { a := vout.Address.String() vin.Addr = a @@ -76,8 +77,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) bchainVout := &bchainTx.Vout[i] vout := &vouts[i] vout.N = i - vout.Value = bchainVout.Value - valOut += bchainVout.Value + vout.ValueSat = bchainVout.ValueSat + vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) + valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses if spendingTx { @@ -85,9 +87,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) } } // for coinbase transactions valIn is 0 - fees = valIn - valOut - if fees < 0 { - fees = 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) } // for now do not return size, we would have to compute vsize of segwit transactions // size:=len(bchainTx.Hex) / 2 @@ -96,13 +98,13 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) Blockheight: int(height), Blocktime: bchainTx.Blocktime, Confirmations: bchainTx.Confirmations, - Fees: fees, + Fees: w.chainParser.AmountToDecimalString(&feesSat), Locktime: bchainTx.LockTime, WithSpends: spendingTx, Time: bchainTx.Time, Txid: bchainTx.Txid, - ValueIn: valIn, - ValueOut: valOut, + ValueIn: w.chainParser.AmountToDecimalString(&valInSat), + ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), Version: bchainTx.Version, Vin: vins, Vout: vouts, @@ -131,26 +133,26 @@ func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return txids, nil } -func (t *Tx) getAddrVoutValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVoutValue(addrID string) *big.Int { + var val big.Int for _, vout := range t.Vout { for _, a := range vout.ScriptPubKey.Addresses { if a == addrID { - val += vout.Value + val.Add(&val, &vout.ValueSat) } } } - return val + return &val } -func (t *Tx) getAddrVinValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVinValue(addrID string) *big.Int { + var val big.Int for _, vin := range t.Vin { if vin.Addr == addrID { - val += vin.Value + val.Add(&val, &vin.ValueSat) } } - return val + return &val } // UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions @@ -191,14 +193,14 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { } txs := make([]*Tx, len(txm)+lc) txi := 0 - var uBal, bal, totRecv, totSent float64 + var uBalSat, balSat, totRecvSat, totSentSat big.Int for _, tx := range txm { tx, err := w.GetTransaction(tx, bestheight, false) // mempool transaction may fail if err != nil { glog.Error("GetTransaction ", tx, ": ", err) } else { - uBal = tx.getAddrVoutValue(addrID) - tx.getAddrVinValue(addrID) + uBalSat.Sub(tx.getAddrVoutValue(addrID), tx.getAddrVinValue(addrID)) txs[txi] = tx txi++ } @@ -216,26 +218,23 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { if err != nil { return nil, err } else { - totRecv += tx.getAddrVoutValue(addrID) - totSent += tx.getAddrVinValue(addrID) + totRecvSat.Add(&totRecvSat, tx.getAddrVoutValue(addrID)) + totSentSat.Add(&totSentSat, tx.getAddrVinValue(addrID)) if i >= from && i < to { txs[txi] = tx txi++ } } } - bal = totRecv - totSent + balSat.Sub(&totRecvSat, &totSentSat) r := &Address{ AddrStr: addrID, - Balance: bal, - BalanceSat: int64(bal*1E8 + 0.5), - TotalReceived: totRecv, - TotalReceivedSat: int64(totRecv*1E8 + 0.5), - TotalSent: totSent, - TotalSentSat: int64(totSent*1E8 + 0.5), + Balance: w.chainParser.AmountToDecimalString(&balSat), + TotalReceived: w.chainParser.AmountToDecimalString(&totRecvSat), + TotalSent: w.chainParser.AmountToDecimalString(&totSentSat), Transactions: txs[:txi], TxApperances: len(txc), - UnconfirmedBalance: uBal, + UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), UnconfirmedTxApperances: len(txm), } glog.Info(addrID, " finished") diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 8f7bc08a..d8d04fef 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -3,6 +3,8 @@ package bchain import ( "encoding/hex" "encoding/json" + "math/big" + "strings" "github.com/gogo/protobuf/proto" "github.com/juju/errors" @@ -14,6 +16,7 @@ type AddressFactoryFunc func(string) (Address, error) type BaseParser struct { AddressFactory AddressFactoryFunc BlockAddressesToKeep int + AmountDecimalPoint int } // AddressToOutputScript converts address to ScriptPubKey - currently not implemented @@ -36,7 +39,47 @@ func (p *BaseParser) ParseTx(b []byte) (*Tx, error) { return nil, errors.New("ParseTx: not implemented") } -// ParseTxFromJson parses JSON message containing transaction and returs Tx struct +const zeros = "0000000000000000000000000000000000000000" + +// AmountToBigInt converts amount in json.Number (string) to big.Int +// it uses string operations to avoid problems with rounding +func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { + var r big.Int + s := string(n) + i := strings.IndexByte(s, '.') + if i == -1 { + s = s + zeros[:p.AmountDecimalPoint] + } else { + z := p.AmountDecimalPoint - len(s) + i + 1 + if z > 0 { + s = s[:i] + s[i+1:] + zeros[:z] + } else { + s = s[:i] + s[i+1:len(s)+z] + } + } + if _, ok := r.SetString(s, 10); !ok { + return r, errors.New("AmountToBigInt: failed to convert") + } + return r, nil +} + +// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place +func (p *BaseParser) AmountToDecimalString(a *big.Int) string { + s := a.String() + if len(s) <= p.AmountDecimalPoint { + s = zeros[:p.AmountDecimalPoint-len(s)+1] + s + } + i := len(s) - p.AmountDecimalPoint + ad := strings.TrimRight(s[i:], "0") + if len(ad) > 0 { + s = s[:i] + "." + ad + } else { + s = s[:i] + } + return s +} + +// ParseTxFromJson parses JSON message containing transaction and returns Tx struct func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { var tx Tx err := json.Unmarshal(msg, &tx) @@ -44,13 +87,18 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { return nil, err } - for i, vout := range tx.Vout { + for i := range tx.Vout { + vout := &tx.Vout[i] + vout.ValueSat, err = p.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } if len(vout.ScriptPubKey.Addresses) == 1 { a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) if err != nil { return nil, err } - tx.Vout[i].Address = a + vout.Address = a } } @@ -127,7 +175,7 @@ func (p *BaseParser) PackTx(tx *Tx, height uint32, blockTime int64) ([]byte, err Addresses: vo.ScriptPubKey.Addresses, N: vo.N, ScriptPubKeyHex: hex, - Value: vo.Value, + ValueSat: vo.ValueSat.Bytes(), } } pt := &ProtoTransaction{ @@ -176,13 +224,15 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } vout := make([]Vout, len(pt.Vout)) for i, pto := range pt.Vout { + var vs big.Int + vs.SetBytes(pto.ValueSat) vout[i] = Vout{ N: pto.N, ScriptPubKey: ScriptPubKey{ Addresses: pto.Addresses, Hex: hex.EncodeToString(pto.ScriptPubKeyHex), }, - Value: pto.Value, + ValueSat: vs, } if len(pto.Addresses) == 1 { a, err := p.AddressFactory(pto.Addresses[0]) diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 68af837a..5dd9b141 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -5,6 +5,7 @@ import ( "blockbook/bchain/coins/btc" "encoding/hex" "encoding/json" + "math/big" "github.com/cpacia/bchutil" "github.com/golang/glog" @@ -132,7 +133,7 @@ func (b *BCashRPC) GetBlockFull(hash string) (*bchain.Block, error) { } // EstimateSmartFee returns fee estimation. -func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := btc.ResEstimateSmartFee{} @@ -141,13 +142,18 @@ func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, err // conservative param is omitted err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } func isErrBlockNotFound(err *bchain.RPCError) bool { diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index b1611161..d9e8eae4 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/big" "reflect" "time" @@ -172,12 +173,12 @@ func (c *blockChainWithMetrics) GetTransactionForMempool(txid string) (v *bchain return c.b.GetTransactionForMempool(txid) } -func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateSmartFee", s, err) }(time.Now()) return c.b.EstimateSmartFee(blocks, conservative) } -func (c *blockChainWithMetrics) EstimateFee(blocks int) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateFee(blocks int) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateFee", s, err) }(time.Now()) return c.b.EstimateFee(blocks) } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 861c7f31..767e8d08 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" vlq "github.com/bsm/go-vlq" "github.com/btcsuite/btcd/blockchain" @@ -30,6 +31,7 @@ func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser &bchain.BaseParser{ AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, params, outputScriptToAddresses, @@ -123,8 +125,10 @@ func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.T // missing: Asm, // missing: Type, } + var vs big.Int + vs.SetInt64(out.Value) vout[i] = bchain.Vout{ - Value: float64(out.Value) / 1E8, + ValueSat: vs, N: uint32(i), ScriptPubKey: s, } diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index ac482b8b..8f9fb067 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -8,6 +8,7 @@ import ( "encoding/json" "io" "io/ioutil" + "math/big" "net" "net/http" "time" @@ -36,18 +37,20 @@ type BitcoinRPC struct { } type Configuration struct { - CoinName string `json:"coin_name"` - RPCURL string `json:"rpcURL"` - RPCUser string `json:"rpcUser"` - RPCPass string `json:"rpcPass"` - RPCTimeout int `json:"rpcTimeout"` - Parse bool `json:"parse"` - ZeroMQBinding string `json:"zeroMQBinding"` - Subversion string `json:"subversion"` - BlockAddressesToKeep int `json:"blockAddressesToKeep"` - MempoolWorkers int `json:"mempoolWorkers"` - MempoolSubWorkers int `json:"mempoolSubWorkers"` - AddressFormat string `json:"addressFormat"` + CoinName string `json:"coin_name"` + RPCURL string `json:"rpcURL"` + RPCUser string `json:"rpcUser"` + RPCPass string `json:"rpcPass"` + RPCTimeout int `json:"rpcTimeout"` + Parse bool `json:"parse"` + ZeroMQBinding string `json:"zeroMQBinding"` + Subversion string `json:"subversion"` + BlockAddressesToKeep int `json:"blockAddressesToKeep"` + MempoolWorkers int `json:"mempoolWorkers"` + MempoolSubWorkers int `json:"mempoolSubWorkers"` + AddressFormat string `json:"addressFormat"` + SupportsEstimateFee bool `json:"supportsEstimateFee"` + SupportsEstimateSmartFee bool `json:"supportsEstimateSmartFee"` } // NewBitcoinRPC returns new BitcoinRPC instance. @@ -69,6 +72,9 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT if c.MempoolSubWorkers < 1 { c.MempoolSubWorkers = 1 } + // btc supports both calls, other coins overriding BitcoinRPC can change this + c.SupportsEstimateFee = true + c.SupportsEstimateSmartFee = true transport := &http.Transport{ Dial: (&net.Dialer{KeepAlive: 600 * time.Second}).Dial, @@ -301,8 +307,8 @@ type CmdEstimateSmartFee struct { type ResEstimateSmartFee struct { Error *bchain.RPCError `json:"error"` Result struct { - Feerate float64 `json:"feerate"` - Blocks int `json:"blocks"` + Feerate json.Number `json:"feerate"` + Blocks int `json:"blocks"` } `json:"result"` } @@ -317,7 +323,7 @@ type CmdEstimateFee struct { type ResEstimateFee struct { Error *bchain.RPCError `json:"error"` - Result float64 `json:"result"` + Result json.Number `json:"result"` } // sendrawtransaction @@ -616,7 +622,12 @@ func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]string, error) { } // EstimateSmartFee returns fee estimation. -func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + // use EstimateFee if EstimateSmartFee is not supported + if !b.ChainConfig.SupportsEstimateSmartFee && b.ChainConfig.SupportsEstimateFee { + return b.EstimateFee(blocks) + } + glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := ResEstimateSmartFee{} @@ -629,17 +640,27 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, e } err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } // EstimateFee returns fee estimation. -func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { +func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { + // use EstimateSmartFee if EstimateFee is not supported + if !b.ChainConfig.SupportsEstimateFee && b.ChainConfig.SupportsEstimateSmartFee { + return b.EstimateSmartFee(blocks, true) + } + glog.V(1).Info("rpc: estimatefee ", blocks) res := ResEstimateFee{} @@ -647,13 +668,18 @@ func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { req.Params.Blocks = blocks err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result, nil + r, err = b.Parser.AmountToBigInt(res.Result) + if err != nil { + return r, err + } + return r, nil } // SendRawTransaction sends raw transaction. diff --git a/bchain/coins/dash/dashrpc.go b/bchain/coins/dash/dashrpc.go index 4c884f08..dec0ee49 100644 --- a/bchain/coins/dash/dashrpc.go +++ b/bchain/coins/dash/dashrpc.go @@ -24,6 +24,7 @@ func NewDashRPC(config json.RawMessage, pushHandler func(bchain.NotificationType b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -53,8 +54,3 @@ func (b *DashRPC) Initialize() error { return nil } - -// EstimateSmartFee returns fee estimation. -func (b *DashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/dogecoin/dogecoinrpc.go b/bchain/coins/dogecoin/dogecoinrpc.go index d0ebb31c..1fc21618 100644 --- a/bchain/coins/dogecoin/dogecoinrpc.go +++ b/bchain/coins/dogecoin/dogecoinrpc.go @@ -24,6 +24,7 @@ func NewDogecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *DogecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *DogecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index fe361258..82d08153 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -22,7 +22,11 @@ type EthereumParser struct { // NewEthereumParser returns new EthereumParser instance func NewEthereumParser() *EthereumParser { - return &EthereumParser{&bchain.BaseParser{AddressFactory: bchain.NewBaseAddress}} + return &EthereumParser{&bchain.BaseParser{ + AddressFactory: bchain.NewBaseAddress, + BlockAddressesToKeep: 0, + AmountDecimalPoint: 18, + }} } type rpcTransaction struct { @@ -86,6 +90,10 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } tx.BlockHash = bh h := hex.EncodeToString(b) + vs, err := hexutil.DecodeBig(tx.Value) + if err != nil { + return nil, err + } return &bchain.Tx{ Blocktime: blocktime, Confirmations: confirmations, @@ -105,8 +113,8 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma }, Vout: []bchain.Vout{ { - N: 0, // there is always up to one To address - // Value - cannot be set, it does not fit precisely to float64 + N: 0, // there is always up to one To address + ValueSat: *vs, ScriptPubKey: bchain.ScriptPubKey{ // Hex Addresses: ta, @@ -286,9 +294,3 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) { func (p *EthereumParser) IsUTXOChain() bool { return false } - -// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column -// do not use the blockaddresses for eth -func (p *EthereumParser) KeepBlockAddresses() int { - return 0 -} diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index d985e74c..2dda7005 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -478,12 +478,12 @@ func (b *EthereumRPC) GetMempool() ([]string, error) { } // EstimateFee returns fee estimation. -func (b *EthereumRPC) EstimateFee(blocks int) (float64, error) { +func (b *EthereumRPC) EstimateFee(blocks int) (big.Int, error) { return b.EstimateSmartFee(blocks, true) } // EstimateSmartFee returns fee estimation. -func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() // TODO - what parameters of msg to use to get better estimate, maybe more data from the wallet are needed @@ -492,10 +492,12 @@ func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, To: &a, } g, err := b.client.EstimateGas(ctx, msg) + var r big.Int if err != nil { - return 0, err + return r, err } - return float64(g), nil + r.SetUint64(g) + return r, nil } // SendRawTransaction sends raw transaction. diff --git a/bchain/coins/litecoin/litecoinrpc.go b/bchain/coins/litecoin/litecoinrpc.go index 0d2ad15f..e7345251 100644 --- a/bchain/coins/litecoin/litecoinrpc.go +++ b/bchain/coins/litecoin/litecoinrpc.go @@ -24,6 +24,7 @@ func NewLitecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *LitecoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *LitecoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/namecoin/namecoinrpc.go b/bchain/coins/namecoin/namecoinrpc.go index faa14fd3..6ab0b0aa 100644 --- a/bchain/coins/namecoin/namecoinrpc.go +++ b/bchain/coins/namecoin/namecoinrpc.go @@ -24,6 +24,7 @@ func NewNamecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *NamecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *NamecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/vertcoin/vertcoinrpc.go b/bchain/coins/vertcoin/vertcoinrpc.go index 878b9682..f282dc78 100644 --- a/bchain/coins/vertcoin/vertcoinrpc.go +++ b/bchain/coins/vertcoin/vertcoinrpc.go @@ -24,6 +24,7 @@ func NewVertcoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *VertcoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *VertcoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 495fa7af..b2c45fe5 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -22,6 +22,7 @@ func NewZCashRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp BitcoinRPC: b.(*btc.BitcoinRPC), } z.RPCMarshaler = btc.JSONMarshalerV1{} + z.ChainConfig.SupportsEstimateSmartFee = false return z, nil } @@ -114,14 +115,6 @@ func (z *ZCashRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { return z.GetTransaction(txid) } -// EstimateSmartFee returns fee estimation. -func (z *ZCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - glog.V(1).Info("rpc: estimatesmartfee") - - // return z.estimateFee(blocks) - return z.EstimateFee(blocks) -} - // GetMempoolEntry returns mempool data for given transaction func (z *ZCashRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { return nil, errors.New("GetMempoolEntry: not implemented") diff --git a/bchain/tx.pb.go b/bchain/tx.pb.go index dc5a0952..2115993a 100644 --- a/bchain/tx.pb.go +++ b/bchain/tx.pb.go @@ -148,7 +148,7 @@ func (m *ProtoTransaction_VinType) GetAddresses() []string { } type ProtoTransaction_VoutType struct { - Value float64 `protobuf:"fixed64,1,opt,name=Value" json:"Value,omitempty"` + ValueSat []byte `protobuf:"bytes,1,opt,name=ValueSat,proto3" json:"ValueSat,omitempty"` N uint32 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` ScriptPubKeyHex []byte `protobuf:"bytes,3,opt,name=ScriptPubKeyHex,proto3" json:"ScriptPubKeyHex,omitempty"` Addresses []string `protobuf:"bytes,4,rep,name=Addresses" json:"Addresses,omitempty"` @@ -159,11 +159,11 @@ func (m *ProtoTransaction_VoutType) String() string { return proto.Co func (*ProtoTransaction_VoutType) ProtoMessage() {} func (*ProtoTransaction_VoutType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 1} } -func (m *ProtoTransaction_VoutType) GetValue() float64 { +func (m *ProtoTransaction_VoutType) GetValueSat() []byte { if m != nil { - return m.Value + return m.ValueSat } - return 0 + return nil } func (m *ProtoTransaction_VoutType) GetN() uint32 { @@ -196,26 +196,26 @@ func init() { func init() { proto.RegisterFile("tx.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 330 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x4e, 0xc2, 0x40, - 0x10, 0x86, 0xb3, 0xb4, 0x14, 0x18, 0x21, 0x92, 0x89, 0x31, 0x0d, 0xf1, 0x50, 0x39, 0xf5, 0xd4, - 0x03, 0xc6, 0x07, 0x50, 0x2f, 0x24, 0x1a, 0x42, 0x16, 0xd2, 0x7b, 0x5b, 0x36, 0xb0, 0x11, 0x77, - 0xb1, 0xdd, 0x1a, 0x78, 0x16, 0x1f, 0xc1, 0x97, 0x34, 0x3b, 0x2d, 0x45, 0x48, 0xbc, 0xed, 0xff, - 0xef, 0x4c, 0xff, 0x6f, 0xff, 0x14, 0xba, 0x66, 0x1f, 0xed, 0x72, 0x6d, 0x34, 0x7a, 0x69, 0xb6, - 0x49, 0xa4, 0x1a, 0x7f, 0xbb, 0x30, 0x9c, 0x5b, 0x67, 0x99, 0x27, 0xaa, 0x48, 0x32, 0x23, 0xb5, - 0x42, 0x04, 0x77, 0xb9, 0x97, 0x2b, 0x9f, 0x05, 0x2c, 0xec, 0x73, 0x3a, 0xe3, 0x10, 0x9c, 0xa9, - 0xd8, 0xfb, 0x2d, 0xb2, 0xec, 0x11, 0xef, 0xa0, 0xf7, 0xbc, 0xd5, 0xd9, 0xbb, 0x91, 0x1f, 0xc2, - 0x77, 0x02, 0x16, 0xba, 0xfc, 0x64, 0xe0, 0x08, 0xba, 0x6f, 0xc7, 0x4b, 0x37, 0x60, 0xe1, 0x80, - 0x37, 0x1a, 0x6f, 0xc1, 0x9b, 0x0a, 0xb9, 0xde, 0x18, 0xbf, 0x4d, 0x37, 0xb5, 0xc2, 0x09, 0x38, - 0xb1, 0x54, 0xbe, 0x17, 0x38, 0xe1, 0xd5, 0x24, 0x88, 0x2a, 0xc4, 0xe8, 0x12, 0x2f, 0x8a, 0xa5, - 0x5a, 0x1e, 0x76, 0x82, 0xdb, 0x61, 0x7c, 0x04, 0x37, 0xd6, 0xa5, 0xf1, 0x3b, 0xb4, 0x74, 0xff, - 0xff, 0x92, 0x2e, 0x0d, 0x6d, 0xd1, 0xf8, 0xe8, 0x87, 0x41, 0xa7, 0xfe, 0x8e, 0x45, 0x7d, 0xd1, - 0x52, 0xa5, 0x49, 0x21, 0xe8, 0xc9, 0x3d, 0xde, 0xe8, 0xa6, 0x8a, 0xd6, 0x9f, 0x2a, 0xb0, 0x8e, - 0x74, 0x08, 0x9e, 0xce, 0x38, 0x86, 0xfe, 0x22, 0xcb, 0xe5, 0xce, 0x2c, 0xe4, 0xda, 0xf6, 0xe4, - 0xd2, 0xfc, 0x99, 0x67, 0x73, 0x16, 0xe2, 0xb3, 0x14, 0x2a, 0x13, 0xf5, 0xc3, 0x1b, 0x6d, 0xcb, - 0x7c, 0x5a, 0xad, 0x72, 0x51, 0x14, 0xa2, 0xa0, 0x02, 0x7a, 0xfc, 0x64, 0x8c, 0xbe, 0xa0, 0x7b, - 0xe4, 0xc7, 0x1b, 0x68, 0xc7, 0xc9, 0xb6, 0xac, 0x50, 0x19, 0xaf, 0x04, 0xf6, 0x81, 0xcd, 0x08, - 0x72, 0xc0, 0xd9, 0x0c, 0x43, 0xb8, 0xae, 0x92, 0xe7, 0x65, 0xfa, 0x2a, 0x0e, 0x16, 0xc8, 0x21, - 0xa0, 0x4b, 0xfb, 0x3c, 0xd7, 0xbd, 0xc8, 0x4d, 0x3d, 0xfa, 0x59, 0x1e, 0x7e, 0x03, 0x00, 0x00, - 0xff, 0xff, 0x64, 0x1d, 0x38, 0xac, 0x38, 0x02, 0x00, 0x00, + // 331 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xc1, 0x6e, 0xea, 0x30, + 0x10, 0x94, 0x89, 0x5f, 0x80, 0x7d, 0xa0, 0x87, 0xf6, 0xf0, 0x14, 0xa1, 0x1e, 0x52, 0x4e, 0x39, + 0xe5, 0x40, 0xd5, 0x0f, 0x68, 0x7b, 0x41, 0x6a, 0x85, 0x90, 0x83, 0x72, 0x4f, 0x82, 0x05, 0x56, + 0xa9, 0x4d, 0x13, 0x47, 0x02, 0xa9, 0x3f, 0xd3, 0x73, 0x7f, 0xb2, 0xf2, 0x12, 0x42, 0x41, 0xea, + 0x6d, 0x67, 0xbc, 0xe3, 0x19, 0x4f, 0x02, 0x3d, 0xbb, 0x8f, 0x77, 0xa5, 0xb1, 0x06, 0xfd, 0xbc, + 0xd8, 0x64, 0x4a, 0x4f, 0x3e, 0x39, 0x8c, 0x16, 0x8e, 0x59, 0x96, 0x99, 0xae, 0xb2, 0xc2, 0x2a, + 0xa3, 0x11, 0x81, 0x2f, 0xf7, 0x6a, 0x15, 0xb0, 0x90, 0x45, 0x03, 0x41, 0x33, 0x8e, 0xc0, 0x9b, + 0xc9, 0x7d, 0xd0, 0x21, 0xca, 0x8d, 0x78, 0x03, 0xfd, 0xc7, 0xad, 0x29, 0x5e, 0xad, 0x7a, 0x93, + 0x81, 0x17, 0xb2, 0x88, 0x8b, 0x33, 0x81, 0x63, 0xe8, 0xbd, 0x9c, 0x0e, 0x79, 0xc8, 0xa2, 0xa1, + 0x68, 0x31, 0xfe, 0x07, 0x7f, 0x26, 0xd5, 0x7a, 0x63, 0x83, 0x3f, 0x74, 0xd2, 0x20, 0x9c, 0x82, + 0x97, 0x2a, 0x1d, 0xf8, 0xa1, 0x17, 0xfd, 0x9d, 0x86, 0xf1, 0x31, 0x62, 0x7c, 0x1d, 0x2f, 0x4e, + 0x95, 0x5e, 0x1e, 0x76, 0x52, 0xb8, 0x65, 0xbc, 0x07, 0x9e, 0x9a, 0xda, 0x06, 0x5d, 0x12, 0xdd, + 0xfe, 0x2e, 0x32, 0xb5, 0x25, 0x15, 0xad, 0x8f, 0xbf, 0x18, 0x74, 0x9b, 0x7b, 0x5c, 0xd4, 0x27, + 0xa3, 0x74, 0x9e, 0x55, 0x92, 0x9e, 0xdc, 0x17, 0x2d, 0x6e, 0xab, 0xe8, 0xfc, 0xa8, 0x02, 0x1b, + 0x4b, 0x8f, 0xc2, 0xd3, 0x8c, 0x13, 0x18, 0x24, 0x45, 0xa9, 0x76, 0x36, 0x51, 0x6b, 0xd7, 0x13, + 0xa7, 0xfd, 0x0b, 0xce, 0xf9, 0x24, 0xf2, 0xbd, 0x96, 0xba, 0x90, 0xcd, 0xc3, 0x5b, 0xec, 0xca, + 0x7c, 0x58, 0xad, 0x4a, 0x59, 0x55, 0xb2, 0xa2, 0x02, 0xfa, 0xe2, 0x4c, 0x8c, 0x3f, 0xa0, 0x77, + 0xca, 0xef, 0x6e, 0x49, 0xb3, 0x6d, 0x2d, 0x93, 0xcc, 0x36, 0x1f, 0xa8, 0xc5, 0x38, 0x00, 0x36, + 0xa7, 0xa8, 0x43, 0xc1, 0xe6, 0x18, 0xc1, 0xbf, 0xa3, 0xff, 0xa2, 0xce, 0x9f, 0xe5, 0xc1, 0xc5, + 0xf2, 0x48, 0x70, 0x4d, 0x5f, 0xba, 0xf3, 0x2b, 0xf7, 0xdc, 0xa7, 0x5f, 0xe6, 0xee, 0x3b, 0x00, + 0x00, 0xff, 0xff, 0x95, 0x9e, 0x00, 0x7a, 0x3e, 0x02, 0x00, 0x00, } diff --git a/bchain/tx.proto b/bchain/tx.proto index 731b8bf5..61e06aed 100644 --- a/bchain/tx.proto +++ b/bchain/tx.proto @@ -11,7 +11,7 @@ syntax = "proto3"; repeated string Addresses = 6; } message VoutType { - double Value = 1; + bytes ValueSat = 1; uint32 N = 2; bytes ScriptPubKeyHex = 3; repeated string Addresses = 4; diff --git a/bchain/types.go b/bchain/types.go index e2aa7276..758f15f9 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math/big" ) // errors with specific meaning returned by blockchain rpc @@ -49,7 +50,8 @@ type Address interface { } type Vout struct { - Value float64 `json:"value"` + ValueSat big.Int + JsonValue json.Number `json:"value"` N uint32 `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` Address Address @@ -90,9 +92,9 @@ type BlockHeader struct { type MempoolEntry struct { Size uint32 `json:"size"` - Fee float64 `json:"fee"` - ModifiedFee float64 `json:"modifiedfee"` - Time float64 `json:"time"` + Fee big.Int `json:"fee"` + ModifiedFee big.Int `json:"modifiedfee"` + Time uint64 `json:"time"` Height uint32 `json:"height"` DescendantCount uint32 `json:"descendantcount"` DescendantSize uint32 `json:"descendantsize"` @@ -132,8 +134,8 @@ type BlockChain interface { GetMempool() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) - EstimateSmartFee(blocks int, conservative bool) (float64, error) - EstimateFee(blocks int) (float64, error) + EstimateSmartFee(blocks int, conservative bool) (big.Int, error) + EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) // mempool ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) @@ -145,7 +147,7 @@ type BlockChain interface { // BlockChainParser defines common interface to parsing and conversions of block chain data type BlockChainParser interface { - // self description + // chain configuration description // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db IsUTXOChain() bool @@ -153,6 +155,11 @@ type BlockChainParser interface { // and used in case of fork // if 0 the blockaddresses column is not used at all (usually non UTXO chains) KeepBlockAddresses() int + // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place + AmountToDecimalString(a *big.Int) string + // AmountToBigInt converts amount in json.Number (string) to big.Int + // it uses string operations to avoid problems with rounding + AmountToBigInt(n json.Number) (big.Int, error) // address id conversions GetAddrIDFromVout(output *Vout) ([]byte, error) GetAddrIDFromAddress(address string) ([]byte, error) diff --git a/server/public.go b/server/public.go index 54555325..acafd08c 100644 --- a/server/public.go +++ b/server/public.go @@ -113,8 +113,10 @@ func formatUnixTime(ut int64) string { return time.Unix(ut, 0).Format(time.RFC1123) } -func formatAmount(a float64) string { - return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%0.8f", a), "0"), ".") +// for now return the string as it is +// in future could be used to do coin specific formatting +func formatAmount(a string) string { + return a } // Run starts the server diff --git a/server/socketio.go b/server/socketio.go index 46c0a563..74fbd67b 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -7,6 +7,7 @@ import ( "blockbook/db" "encoding/json" "net/http" + "strconv" "strings" "time" @@ -236,11 +237,11 @@ type txInputs struct { // ScriptAsm *string `json:"scriptAsm"` Sequence int64 `json:"sequence"` Address *string `json:"address"` - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` } type txOutputs struct { - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` Script *string `json:"script"` // ScriptAsm *string `json:"scriptAsm"` SpentTxID *string `json:"spentTxId,omitempty"` @@ -267,7 +268,7 @@ type resTx struct { type addressHistoryItem struct { Addresses map[string]addressHistoryIndexes `json:"addresses"` - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` Confirmations int `json:"confirmations"` Tx resTx `json:"tx"` } @@ -329,7 +330,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r for _, vout := range tx.Vout { aoh := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.String(), Script: &aoh, } if vout.Address != nil { @@ -452,6 +453,7 @@ func unmarshalEstimateSmartFee(params []byte) (blocks int, conservative bool, er } type resultEstimateSmartFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -460,7 +462,7 @@ func (s *SocketIoServer) estimateSmartFee(blocks int, conservative bool) (res re if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -479,6 +481,7 @@ func unmarshalEstimateFee(params []byte) (blocks int, err error) { } type resultEstimateFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -487,7 +490,7 @@ func (s *SocketIoServer) estimateFee(blocks int) (res resultEstimateFee, err err if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -588,7 +591,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai a := vout.Address.String() ai.Address = &a } - ai.Satoshis = int64(vout.Value*1E8 + 0.5) + ai.Satoshis = vout.ValueSat.String() } } hi = append(hi, ai) @@ -596,7 +599,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai for _, vout := range tx.Vout { aos := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.String(), Script: &aos, } if vout.Address != nil { From 1ae62cc974503b21dbae66f046f10a05fe06df63 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 24 Jul 2018 21:21:05 +0200 Subject: [PATCH 02/18] Fix coin parser tests after switch to big.Int amounts --- bchain/coins/bch/bcashparser_test.go | 13 +- bchain/coins/btc/bitcoinparser_test.go | 13 +- bchain/coins/dogecoin/dogecoinparser_test.go | 17 +- bchain/coins/eth/ethparser_test.go | 173 +++++++++---------- bchain/coins/litecoin/litecoinparser_test.go | 9 +- bchain/coins/vertcoin/vertcoinparser_test.go | 9 +- bchain/coins/zec/zcashparser_test.go | 17 +- 7 files changed, 119 insertions(+), 132 deletions(-) diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index 3e1cb4c1..d6040b82 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -7,6 +7,7 @@ import ( "blockbook/bchain/coins/btc" "bytes" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -159,8 +160,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ @@ -190,8 +191,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ @@ -201,8 +202,8 @@ func init() { Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index d605f45b..79d02816 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -5,6 +5,7 @@ package btc import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -152,8 +153,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ @@ -183,8 +184,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ @@ -194,8 +195,8 @@ func init() { Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ diff --git a/bchain/coins/dogecoin/dogecoinparser_test.go b/bchain/coins/dogecoin/dogecoinparser_test.go index b8ea585f..0b736c51 100644 --- a/bchain/coins/dogecoin/dogecoinparser_test.go +++ b/bchain/coins/dogecoin/dogecoinparser_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "path/filepath" "reflect" "testing" @@ -111,8 +112,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 27478.75452951, - N: 0, + ValueSat: *big.NewInt(2747875452951), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914eef21768a546590993e313c7f3dfadf6a6efa1e888ac", Addresses: []string{ @@ -122,8 +123,8 @@ func init() { Address: addr1, }, { - Value: 74.20567469, - N: 1, + ValueSat: *big.NewInt(7420567469), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e0fee2ea29dd9c6c759d8341bd0da4c4f738cced88ac", Addresses: []string{ @@ -153,8 +154,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 59890867.89818935, - N: 0, + ValueSat: *big.NewInt(5989086789818935), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149355c01ed20057eac9fe0bbf8b07d87e62fe712d88ac", Addresses: []string{ @@ -164,8 +165,8 @@ func init() { Address: addr3, }, { - Value: 9999998.90000000, - N: 1, + ValueSat: *big.NewInt(999999890000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9145b4f2511c94e4fcaa8f8835b2458f8cb6542ca7688ac", Addresses: []string{ diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index c1f4620f..40069a55 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -5,6 +5,7 @@ package eth import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -63,6 +64,68 @@ func TestEthParser_GetAddrIDFromAddress(t *testing.T) { } } +var ( + testTx1, testTx2 bchain.Tx + testTxPacked1 = "08aebf0a1205012a05f20018a0f73622081234567890abcdef2a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f" + testTxPacked2 = "08ece40212050430e234001888a4012201213220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915" +) + +func init() { + var ( + addr1, addr2 bchain.Address + err error + ) + addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") + if err == nil { + addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") + } + if err != nil { + panic(err) + } + + testTx1 = bchain.Tx{ + Blocktime: 1521515026, + Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307831323334353637383930616263646566222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", + Time: 1521515026, + Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + Vin: []bchain.Vin{ + { + Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(1311768467294899695), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, + }, + Address: addr1, + }, + }, + } + + testTx2 = bchain.Tx{ + Blocktime: 1521533434, + Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a2230783231222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", + Time: 1521533434, + Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", + Vin: []bchain.Vin{ + { + Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(33), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, + }, + Address: addr2, + }, + }, + } +} + func TestEthereumParser_PackTx(t *testing.T) { type args struct { tx *bchain.Tx @@ -77,61 +140,27 @@ func TestEthereumParser_PackTx(t *testing.T) { wantErr bool }{ { - name: "with 0x prefix", + name: "1", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - }, - }, - }, + tx: &testTx1, height: 2870000, blockTime: 1521515026, }, - want: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f", + want: testTxPacked1, }, { - name: "without 0x prefix", + name: "2", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - }, - }, - }, + tx: &testTx2, height: 2871048, blockTime: 1521533434, }, - want: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915", + want: testTxPacked2, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime) if (err != nil) != tt.wantErr { t.Errorf("EthereumParser.PackTx() error = %v, wantErr %v", err, tt.wantErr) @@ -146,18 +175,6 @@ func TestEthereumParser_PackTx(t *testing.T) { } func TestEthereumParser_UnpackTx(t *testing.T) { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") - if err == nil { - addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") - } - if err != nil { - panic(err) - } - type args struct { hex string } @@ -170,57 +187,21 @@ func TestEthereumParser_UnpackTx(t *testing.T) { wantErr bool }{ { - name: "1", - args: args{hex: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f"}, - want: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - Address: addr1, - }, - }, - }, + name: "1", + args: args{hex: testTxPacked1}, + want: &testTx1, want1: 2870000, }, { - name: "1", - args: args{hex: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915"}, - want: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - Address: addr2, - }, - }, - }, + name: "2", + args: args{hex: testTxPacked2}, + want: &testTx2, want1: 2871048, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() b, err := hex.DecodeString(tt.args.hex) if err != nil { panic(err) diff --git a/bchain/coins/litecoin/litecoinparser_test.go b/bchain/coins/litecoin/litecoinparser_test.go index 11c55f1e..a76faf63 100644 --- a/bchain/coins/litecoin/litecoinparser_test.go +++ b/bchain/coins/litecoin/litecoinparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -173,8 +174,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 12.52000000, - N: 0, + ValueSat: *big.NewInt(1252000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9141ae882e788091732da6910595314447c9e38bd8d88ac", Addresses: []string{ @@ -184,8 +185,8 @@ func init() { Address: addr1, }, { - Value: 0.01000487, - N: 1, + ValueSat: *big.NewInt(1000487), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9146b474cbf0f6004329b630bdd4798f2c23d1751b688ac", Addresses: []string{ diff --git a/bchain/coins/vertcoin/vertcoinparser_test.go b/bchain/coins/vertcoin/vertcoinparser_test.go index 558ae51f..0cbf01d5 100644 --- a/bchain/coins/vertcoin/vertcoinparser_test.go +++ b/bchain/coins/vertcoin/vertcoinparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -105,8 +106,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.10781192, - N: 0, + ValueSat: *big.NewInt(10781192), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a91499b16da88a7e29b913b6131df2644d6d06cb331b88ac", Addresses: []string{ @@ -116,8 +117,8 @@ func init() { Address: addr1, }, { - Value: 0.5000000, - N: 1, + ValueSat: *big.NewInt(50000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a91446eb90e002f137f05385896c882fe000cc2e967f87", Addresses: []string{ diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index 0bb61fe1..b1bfd773 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -13,8 +14,8 @@ import ( var ( testTx1, testTx2 bchain.Tx - testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a4b091e6c90cd3ebc664010001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" - testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a4b09257b2170264d504010001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a4b099a9999999999b93f10011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" + testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a490a05043c1aec8e10001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" + testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a490a050184a7a72310001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a470a0398968010011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" ) func init() { @@ -51,8 +52,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 181.88266638, - N: 0, + ValueSat: *big.NewInt(18188266638), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac", Addresses: []string{ @@ -82,8 +83,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 65.20547107, - N: 0, + ValueSat: *big.NewInt(6520547107), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914826f87806ddd4643730be99b41c98acc379e83db88ac", Addresses: []string{ @@ -93,8 +94,8 @@ func init() { Address: addr2, }, { - Value: .1, - N: 1, + ValueSat: *big.NewInt(10000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e395634b7684289285926d4c64db395b783720ec88ac", Addresses: []string{ From f80a9142d9becee2a884323e1a49e48c3804c446 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 25 Jul 2018 11:47:07 +0200 Subject: [PATCH 03/18] Handle negative big.Int amounts, add tests of amounts parser --- bchain/baseparser.go | 21 +++++++++------ bchain/baseparser_test.go | 56 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 bchain/baseparser_test.go diff --git a/bchain/baseparser.go b/bchain/baseparser.go index d8d04fef..ce6dc947 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -65,18 +65,23 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place func (p *BaseParser) AmountToDecimalString(a *big.Int) string { - s := a.String() - if len(s) <= p.AmountDecimalPoint { - s = zeros[:p.AmountDecimalPoint-len(s)+1] + s + n := a.String() + var s string + if n[0] == '-' { + n = n[1:] + s = "-" } - i := len(s) - p.AmountDecimalPoint - ad := strings.TrimRight(s[i:], "0") + if len(n) <= p.AmountDecimalPoint { + n = zeros[:p.AmountDecimalPoint-len(n)+1] + n + } + i := len(n) - p.AmountDecimalPoint + ad := strings.TrimRight(n[i:], "0") if len(ad) > 0 { - s = s[:i] + "." + ad + n = n[:i] + "." + ad } else { - s = s[:i] + n = n[:i] } - return s + return s + n } // ParseTxFromJson parses JSON message containing transaction and returns Tx struct diff --git a/bchain/baseparser_test.go b/bchain/baseparser_test.go new file mode 100644 index 00000000..cfb20a68 --- /dev/null +++ b/bchain/baseparser_test.go @@ -0,0 +1,56 @@ +package bchain + +import ( + "encoding/json" + "math/big" + "testing" +) + +func NewBaseParser(adp int) *BaseParser { + return &BaseParser{ + AmountDecimalPoint: adp, + } +} + +var amounts = []struct { + a *big.Int + s string + adp int + alternative string +}{ + {big.NewInt(123456789), "1.23456789", 8, "!"}, + {big.NewInt(2), "0.00000002", 8, "!"}, + {big.NewInt(300000000), "3", 8, "!"}, + {big.NewInt(498700000), "4.987", 8, "!"}, + {big.NewInt(567890), "0.00000000000056789", 18, "!"}, + {big.NewInt(-100000000), "-1", 8, "!"}, + {big.NewInt(-8), "-0.00000008", 8, "!"}, + {big.NewInt(-89012345678), "-890.12345678", 8, "!"}, + {big.NewInt(-12345), "-0.00012345", 8, "!"}, + {big.NewInt(12345678), "0.123456789012", 8, "0.12345678"}, // test of truncation of too many decimal places +} + +func TestBaseParser_AmountToDecimalString(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + if got := NewBaseParser(tt.adp).AmountToDecimalString(tt.a); got != tt.s && got != tt.alternative { + t.Errorf("BaseParser.AmountToDecimalString() = %v, want %v", got, tt.s) + } + }) + } +} + +func TestBaseParser_AmountToBigInt(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + got, err := NewBaseParser(tt.adp).AmountToBigInt(json.Number(tt.s)) + if err != nil { + t.Errorf("BaseParser.AmountToBigInt() error = %v", err) + return + } + if got.Cmp(tt.a) != 0 { + t.Errorf("BaseParser.AmountToBigInt() = %v, want %v", got, tt.a) + } + }) + } +} From 882ff1bfd5adedafbfed89d388debd06aa94282a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 25 Jul 2018 15:56:08 +0200 Subject: [PATCH 04/18] Fix integration tests after switch to big.Int amounts --- bchain/baseparser.go | 2 ++ bchain/coins/bch/bcashparser.go | 1 + bchain/coins/btc/bitcoinrpc.go | 9 +++++- bchain/coins/zec/zcashparser.go | 3 +- bchain/tests/rpc/data.go | 16 ++++++++-- bchain/tests/rpc/rpc.go | 28 +++++++++++------- bchain/tests/rpc/testdata/Bcash.json | 2 ++ bchain/tests/rpc/testdata/Bcash_Testnet.json | 2 ++ bchain/tests/rpc/testdata/Bitcoin.json | 2 ++ .../tests/rpc/testdata/Bitcoin_Testnet.json | 2 ++ bchain/tests/rpc/testdata/Dash.json | 1 + bchain/tests/rpc/testdata/Dash_Testnet.json | 2 ++ .../tests/rpc/testdata/Ethereum_Testnet.json | 29 ++++--------------- bchain/tests/rpc/testdata/Namecoin.json | 1 + bchain/tests/rpc/testdata/Vertcoin.json | 2 ++ bchain/tests/rpc/testdata/Zcash.json | 2 ++ bchain/tests/rpc/testdata/Zcash_Testnet.json | 2 ++ bchain/types.go | 26 +++++++++-------- 18 files changed, 81 insertions(+), 51 deletions(-) diff --git a/bchain/baseparser.go b/bchain/baseparser.go index ce6dc947..172a8180 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -94,10 +94,12 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { for i := range tx.Vout { vout := &tx.Vout[i] + // convert vout.JsonValue to big.Int and clear it, it is only temporary value used for unmarshal vout.ValueSat, err = p.AmountToBigInt(vout.JsonValue) if err != nil { return nil, err } + vout.JsonValue = "" if len(vout.ScriptPubKey.Addresses) == 1 { a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) if err != nil { diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 95284d84..02286ca6 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -49,6 +49,7 @@ func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) (*BCashParser BaseParser: &bchain.BaseParser{ AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, format) }, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, Params: params, OutputScriptToAddressesFunc: outputScriptToAddresses, diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index 8f9fb067..c8a9f476 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -710,13 +710,20 @@ func (b *BitcoinRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) Params: []string{txid}, } err := b.Call(&req, &res) - if err != nil { return nil, err } if res.Error != nil { return nil, res.Error } + res.Result.FeeSat, err = b.Parser.AmountToBigInt(res.Result.Fee) + if err != nil { + return nil, err + } + res.Result.ModifiedFeeSat, err = b.Parser.AmountToBigInt(res.Result.ModifiedFee) + if err != nil { + return nil, err + } return res.Result, nil } diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index eacf8bbd..dc2d84d9 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -20,12 +20,13 @@ type ZCashParser struct { *bchain.BaseParser } -// NewZCAshParser returns new ZCAshParser instance +// NewZCashParser returns new ZCashParser instance func NewZCashParser(c *btc.Configuration) *ZCashParser { return &ZCashParser{ &bchain.BaseParser{ AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, } } diff --git a/bchain/tests/rpc/data.go b/bchain/tests/rpc/data.go index 7d50460a..68d69acf 100644 --- a/bchain/tests/rpc/data.go +++ b/bchain/tests/rpc/data.go @@ -1,8 +1,7 @@ -// +build integration - package rpc import ( + "blockbook/bchain" "encoding/json" "errors" "fmt" @@ -83,7 +82,7 @@ func LoadRPCConfig(coin string) (json.RawMessage, error) { return json.RawMessage(fmt.Sprintf(t, coin, c.URL, c.User, c.Pass)), nil } -func LoadTestData(coin string) (*TestData, error) { +func LoadTestData(coin string, parser bchain.BlockChainParser) (*TestData, error) { b, err := readDataFile(".", "bchain/tests/rpc/testdata", coin+".json") if err != nil { return nil, err @@ -93,5 +92,16 @@ func LoadTestData(coin string) (*TestData, error) { if err != nil { return nil, err } + // convert amounts in test json to bit.Int and clear the temporary JsonValue + for _, tx := range v.TxDetails { + for i := range tx.Vout { + vout := &tx.Vout[i] + vout.ValueSat, err = parser.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } + vout.JsonValue = "" + } + } return &v, nil } diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go index 4386baa1..20483573 100644 --- a/bchain/tests/rpc/rpc.go +++ b/bchain/tests/rpc/rpc.go @@ -1,5 +1,3 @@ -// +build integration - package rpc import ( @@ -9,6 +7,7 @@ import ( "net" "reflect" "testing" + "time" "github.com/deckarep/golang-set" ) @@ -56,7 +55,7 @@ func NewTest(coin string, factory TestChainFactoryFunc) (*Test, error) { return nil, err } } else { - td, err = LoadTestData(coin) + td, err = LoadTestData(coin, cli.GetChainParser()) if err != nil { return nil, err } @@ -162,7 +161,7 @@ func (rt *Test) TestGetTransaction(t *testing.T) { got.Confirmations = 0 if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransaction() got %v, want %v", got, want) + t.Errorf("GetTransaction() got %+v, want %+v", got, want) } } } @@ -181,7 +180,7 @@ func (rt *Test) TestGetTransactionForMempool(t *testing.T) { // transactions parsed from JSON may contain additional data got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransactionForMempool() got %v, want %v", got, want) + t.Errorf("GetTransactionForMempool() got %+v, want %+v", got, want) } } } @@ -329,8 +328,8 @@ func (rt *Test) TestGetMempoolEntry(t *testing.T) { if e.Size <= 0 { t.Errorf("GetMempoolEntry() got zero or negative size %d", e.Size) } - if e.Fee <= 0 { - t.Errorf("GetMempoolEntry() got zero or negative fee %f", e.Fee) + if e.FeeSat.Sign() != 1 { + t.Errorf("GetMempoolEntry() got zero or negative fee %v", e.FeeSat.String()) } // done @@ -347,8 +346,11 @@ func (rt *Test) TestEstimateSmartFee(t *testing.T) { if err != nil { t.Error(err) } - if fee != -1 && fee < 0 { - t.Errorf("EstimateSmartFee() returned unexpected fee rate: %f", fee) + if fee.Sign() == -1 { + sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateSmartFee() returned unexpected fee rate: %v", sf) + } } } } @@ -361,8 +363,11 @@ func (rt *Test) TestEstimateFee(t *testing.T) { if err != nil { t.Error(err) } - if fee != -1 && fee < 0 { - t.Errorf("EstimateFee() returned unexpected fee rate: %f", fee) + if fee.Sign() == -1 { + sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateFee() returned unexpected fee rate: %v", sf) + } } } } @@ -385,6 +390,7 @@ func (rt *Test) TestGetBestBlockHash(t *testing.T) { t.Fatal(err) } if hash != hh { + time.Sleep(time.Millisecond * 100) continue } diff --git a/bchain/tests/rpc/testdata/Bcash.json b/bchain/tests/rpc/testdata/Bcash.json index f72df31f..9b1f647b 100644 --- a/bchain/tests/rpc/testdata/Bcash.json +++ b/bchain/tests/rpc/testdata/Bcash.json @@ -70,6 +70,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "fe496933e0a3582ef020bd35c38fe8244a80fa7c63e7607f6f9ccb0806f419e4", @@ -175,6 +176,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "52f37874fd5a2497c5d84619ab415ca17ebb1d49d559f0d468869f70537354b9", diff --git a/bchain/tests/rpc/testdata/Bcash_Testnet.json b/bchain/tests/rpc/testdata/Bcash_Testnet.json index 39eba5e7..c33176d3 100644 --- a/bchain/tests/rpc/testdata/Bcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Bcash_Testnet.json @@ -12,6 +12,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "coinbase": "03fbf212055374617368", @@ -34,6 +35,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "txid": "58ff2450a71b3c228d17d04ec8edcaae452bc97d5ccc4591e2741fd8b031d221", diff --git a/bchain/tests/rpc/testdata/Bitcoin.json b/bchain/tests/rpc/testdata/Bitcoin.json index ba0467cc..5cf34ec8 100644 --- a/bchain/tests/rpc/testdata/Bitcoin.json +++ b/bchain/tests/rpc/testdata/Bitcoin.json @@ -120,6 +120,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 0, + "version": 1, "vin": [ { "txid": "83eff5bcc738d52e04528984cd5cf601b69e7df65b2b10ed2475c0072cdde14c", @@ -153,6 +154,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 529149, + "version": 2, "vin": [ { "txid": "413988d546516707f3b20ca9876e026a9472bd814662e42f37f4e57a15713da7", diff --git a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json index ece3c941..cab2c840 100644 --- a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json +++ b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json @@ -43,6 +43,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "4097f5265397047bffe219a1bca65ea5726402c3d9aeecd577fa1c274fc1b8a4", @@ -84,6 +85,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "80437ba4e81bde8b584258f5adad0d08dea60f1e38026344442ad59a4ef797c9", diff --git a/bchain/tests/rpc/testdata/Dash.json b/bchain/tests/rpc/testdata/Dash.json index 150e7d51..f5ab09b6 100644 --- a/bchain/tests/rpc/testdata/Dash.json +++ b/bchain/tests/rpc/testdata/Dash.json @@ -27,6 +27,7 @@ "blocktime": 1530189699, "time": 1530189699, "locktime": 0, + "version": 1, "vin": [ { "txid": "a78824a88c089ea1344fe2a33c454024e6501e76cca2590515b390d85082bd23", diff --git a/bchain/tests/rpc/testdata/Dash_Testnet.json b/bchain/tests/rpc/testdata/Dash_Testnet.json index d86f0bee..64509772 100644 --- a/bchain/tests/rpc/testdata/Dash_Testnet.json +++ b/bchain/tests/rpc/testdata/Dash_Testnet.json @@ -31,6 +31,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "a41616c7585c98aeda98d6ff6766b15455e327c9472582b80289dab7597ad309", @@ -64,6 +65,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "187ae015e41dff766f5a18ae705f59db950a6729a06fa5fd04630c362a9aee27", diff --git a/bchain/tests/rpc/testdata/Ethereum_Testnet.json b/bchain/tests/rpc/testdata/Ethereum_Testnet.json index 82bbf3c7..b7cd76cb 100644 --- a/bchain/tests/rpc/testdata/Ethereum_Testnet.json +++ b/bchain/tests/rpc/testdata/Ethereum_Testnet.json @@ -16,38 +16,21 @@ "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0" ], "txDetails": { - "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d": { - "hex": "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - "txid": "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0": { + "hex": "7b226e6f6e6365223a223078333632306136222c226761735072696365223a223078313261303566323030222c22676173223a22307835323038222c22746f223a22307831623137626331326166623635643563346238316139666632613037366234326131396661616136222c2276616c7565223a223078646530623662336137363430303030222c22696e707574223a223078222c2268617368223a22307837663064313430333239393431663132306235623366633735316533306164656238376232616562626663653561646364303231363630346133346236636330222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838316237653038663635626466353634383630366338393939386139636338313634333937363437222c227472616e73616374696f6e496e646578223a22307862222c2276223a2230783162222c2272223a22307862666662323864633865373939383833366639356664616139396433626435666635346365663562313839636462383537333537666161326431616231393136222c2273223a22307833616462316365313264396664306538616662373064386639346534636538356137666639346465333966623636333139363638363435663464643138646432227d", + "txid": "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0", "blocktime": 1521515026, "time": 1521515026, "vin": [ { - "addresses": ["0xdacc9c61754a0c4616fc5323dc946e89eb272302"] + "addresses": ["0x81b7e08f65bdf5648606c89998a9cc8164397647"] } ], "vout": [ { + "value": 1, "scriptPubKey": { - "addresses": ["0x682b7903a11098cf770c7aef4aa02a85b3f3601a"] - } - } - ] - }, - "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3": { - "hex": "7b226e6f6e6365223a223078346566222c226761735072696365223a223078626134336237343030222c22676173223a2230783936653838222c22746f223a22307864303534323939633438326164356362333961336661383734373235663965326132306433343963222c2276616c7565223a22307830222c22696e707574223a2230783165626366613630303030303030303030303030303030303030303030303030633034383732383566313736663532306635636434363732306236383662366534366165636430373030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303031303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030313430303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030633461303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303530303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030353831303030303030303030303030303030303030303030303030633365313631343261306432366336616530623163656238643961373236653734653164613664663666656562653137653437306464626434656261663364623938333136386235316337623136363732613131623966306662383864623438323962653237346230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303034363336663665363230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303437323666373336313030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030222c2268617368223a22307831376565323335666330333539313535623235343139653065346336356439633530306466366537316538323838643665663032306430346363326632636233222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838623461353365353739303739343466633261653539623837373066393331623639326232373062222c227472616e73616374696f6e496e646578223a22307830222c2276223a2230783163222c2272223a22307866633739633836353638323039313030323134616339353662373930653066383935656130343135313438643233613239353632356564393761633936333534222c2273223a223078373232303833616331643764663662626162393939383537616163323561336665646265386130633561326266376364653835393738363266373862313937227d", - "txid": "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", - "blocktime": 1521515026, - "time": 1521515026, - "vin": [ - { - "addresses": ["0x8b4a53e57907944fc2ae59b8770f931b692b270b"] - } - ], - "vout": [ - { - "scriptPubKey": { - "addresses": ["0xd054299c482ad5cb39a3fa874725f9e2a20d349c"] + "addresses": ["0x1b17bc12afb65d5c4b81a9ff2a076b42a19faaa6"] } } ] diff --git a/bchain/tests/rpc/testdata/Namecoin.json b/bchain/tests/rpc/testdata/Namecoin.json index 4e1193cd..5a56c972 100644 --- a/bchain/tests/rpc/testdata/Namecoin.json +++ b/bchain/tests/rpc/testdata/Namecoin.json @@ -39,6 +39,7 @@ "blocktime": 1530003649, "time": 1530003649, "locktime": 404678, + "version": 1, "vin": [ { "txid": "223a4ff7a613173ffbc67f9efffe1b0c3fd5ba2471935a44cc81814038272901", diff --git a/bchain/tests/rpc/testdata/Vertcoin.json b/bchain/tests/rpc/testdata/Vertcoin.json index 021cdc2a..d5c5383e 100644 --- a/bchain/tests/rpc/testdata/Vertcoin.json +++ b/bchain/tests/rpc/testdata/Vertcoin.json @@ -14,6 +14,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952232, + "version": 2, "vin": [ { "txid": "8ec70404bdd16559c589736863ee9d9f69431ba4e6b2ab3b1641b3f9f3821624", @@ -47,6 +48,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952233, + "version": 1, "vin": [ { "txid": "5765db7dd62cddada154b2c95505e298818a308cc1ce1e28d6e8565301cb8d74", diff --git a/bchain/tests/rpc/testdata/Zcash.json b/bchain/tests/rpc/testdata/Zcash.json index 6b620537..346d9f96 100644 --- a/bchain/tests/rpc/testdata/Zcash.json +++ b/bchain/tests/rpc/testdata/Zcash.json @@ -22,6 +22,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 349399, + "version": 3, "vin": [ { "txid": "9c9faac29b0fa1b0e683727f2973bfcb87e4baadd07e9c997b431a31713cb30c", @@ -57,6 +58,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 0, + "version": 3, "vin": [ { "txid": "4440c8e9d5b57da7ca0fb3f62ec13f392267aeb797c123bb01e850adf8573dd0", diff --git a/bchain/tests/rpc/testdata/Zcash_Testnet.json b/bchain/tests/rpc/testdata/Zcash_Testnet.json index aca84131..5d78989c 100644 --- a/bchain/tests/rpc/testdata/Zcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Zcash_Testnet.json @@ -13,6 +13,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251028, + "version": 3, "vin": [ { "txid": "19a1d013b898239e9a2943faa07f8716b9be168bc8e001daf3625f535fde1a60", @@ -48,6 +49,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251090, + "version": 3, "vin": [ { "txid": "9acab5f13cf94074e75f5686b59fccd938f54b5f20ddddfcb6077c679a13c0ea", diff --git a/bchain/types.go b/bchain/types.go index 758f15f9..cfe90771 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -91,18 +91,20 @@ type BlockHeader struct { } type MempoolEntry struct { - Size uint32 `json:"size"` - Fee big.Int `json:"fee"` - ModifiedFee big.Int `json:"modifiedfee"` - Time uint64 `json:"time"` - Height uint32 `json:"height"` - DescendantCount uint32 `json:"descendantcount"` - DescendantSize uint32 `json:"descendantsize"` - DescendantFees uint32 `json:"descendantfees"` - AncestorCount uint32 `json:"ancestorcount"` - AncestorSize uint32 `json:"ancestorsize"` - AncestorFees uint32 `json:"ancestorfees"` - Depends []string `json:"depends"` + Size uint32 `json:"size"` + FeeSat big.Int + Fee json.Number `json:"fee"` + ModifiedFeeSat big.Int + ModifiedFee json.Number `json:"modifiedfee"` + Time uint64 `json:"time"` + Height uint32 `json:"height"` + DescendantCount uint32 `json:"descendantcount"` + DescendantSize uint32 `json:"descendantsize"` + DescendantFees uint32 `json:"descendantfees"` + AncestorCount uint32 `json:"ancestorcount"` + AncestorSize uint32 `json:"ancestorsize"` + AncestorFees uint32 `json:"ancestorfees"` + Depends []string `json:"depends"` } type RPCError struct { From d6cbe7c513623f7e56851ab3e799e960fa2799f0 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 26 Jul 2018 00:15:29 +0200 Subject: [PATCH 05/18] Remove duplicate mempool transactions --- api/worker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/worker.go b/api/worker.go index ea2870e6..01064560 100644 --- a/api/worker.go +++ b/api/worker.go @@ -183,6 +183,7 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { if err != nil { return nil, err } + txm = UniqueTxidsInReverse(txm) bestheight, _, err := w.db.GetBestBlock() if err != nil { return nil, err From 75d48376e14f6fb19b1af3e80759518ee86183e9 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 27 Jul 2018 14:08:20 +0200 Subject: [PATCH 06/18] Change db columns, enable compression on some columns --- db/rocksdb.go | 73 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 30285dab..885b57d1 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -46,12 +46,17 @@ const ( cfDefault = iota cfHeight cfAddresses - cfUnspentTxs + cfTxAddresses + cfAddressBalance + cfBlockTxids cfTransactions + + // to be removed, kept temporarily so that the code is compilable + cfUnspentTxs cfBlockAddresses ) -var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"} +var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -61,37 +66,49 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) bbto.SetBlockCache(c) bbto.SetFilterPolicy(fp) - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) - opts.SetMaxBackgroundCompactions(4) - opts.SetMaxBackgroundFlushes(2) - opts.SetBytesPerSync(1 << 20) // 1MB - opts.SetWriteBufferSize(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) - opts.SetCompression(gorocksdb.NoCompression) + optsNoCompression := gorocksdb.NewDefaultOptions() + optsNoCompression.SetBlockBasedTableFactory(bbto) + optsNoCompression.SetCreateIfMissing(true) + optsNoCompression.SetCreateIfMissingColumnFamilies(true) + optsNoCompression.SetMaxBackgroundCompactions(4) + optsNoCompression.SetMaxBackgroundFlushes(2) + optsNoCompression.SetBytesPerSync(1 << 20) // 1MB + optsNoCompression.SetWriteBufferSize(1 << 27) // 128MB + optsNoCompression.SetMaxOpenFiles(25000) + optsNoCompression.SetCompression(gorocksdb.NoCompression) - // opts for outputs are different: + optsLZ4 := gorocksdb.NewDefaultOptions() + optsLZ4.SetBlockBasedTableFactory(bbto) + optsLZ4.SetCreateIfMissing(true) + optsLZ4.SetCreateIfMissingColumnFamilies(true) + optsLZ4.SetMaxBackgroundCompactions(4) + optsLZ4.SetMaxBackgroundFlushes(2) + optsLZ4.SetBytesPerSync(1 << 20) // 1MB + optsLZ4.SetWriteBufferSize(1 << 27) // 128MB + optsLZ4.SetMaxOpenFiles(25000) + optsLZ4.SetCompression(gorocksdb.LZ4Compression) + + // opts for addresses are different: // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter - bbtoOutputs := gorocksdb.NewDefaultBlockBasedTableOptions() - bbtoOutputs.SetBlockSize(16 << 10) // 16kB - bbtoOutputs.SetBlockCache(c) // 8GB + bbtoAddresses := gorocksdb.NewDefaultBlockBasedTableOptions() + bbtoAddresses.SetBlockSize(16 << 10) // 16kB + bbtoAddresses.SetBlockCache(c) // 8GB - optsOutputs := gorocksdb.NewDefaultOptions() - optsOutputs.SetBlockBasedTableFactory(bbtoOutputs) - optsOutputs.SetCreateIfMissing(true) - optsOutputs.SetCreateIfMissingColumnFamilies(true) - optsOutputs.SetMaxBackgroundCompactions(4) - optsOutputs.SetMaxBackgroundFlushes(2) - optsOutputs.SetBytesPerSync(1 << 20) // 1MB - optsOutputs.SetWriteBufferSize(1 << 27) // 128MB - optsOutputs.SetMaxOpenFiles(25000) - optsOutputs.SetCompression(gorocksdb.NoCompression) + optsAddresses := gorocksdb.NewDefaultOptions() + optsAddresses.SetBlockBasedTableFactory(bbtoAddresses) + optsAddresses.SetCreateIfMissing(true) + optsAddresses.SetCreateIfMissingColumnFamilies(true) + optsAddresses.SetMaxBackgroundCompactions(4) + optsAddresses.SetMaxBackgroundFlushes(2) + optsAddresses.SetBytesPerSync(1 << 20) // 1MB + optsAddresses.SetWriteBufferSize(1 << 27) // 128MB + optsAddresses.SetMaxOpenFiles(25000) + optsAddresses.SetCompression(gorocksdb.NoCompression) - fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts} + // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions + fcOptions := []*gorocksdb.Options{optsNoCompression, optsNoCompression, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) + db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions) if err != nil { return nil, nil, err } From e558c10da927c8ccb99d10093bcec1078fc8b778 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 27 Jul 2018 19:46:21 +0200 Subject: [PATCH 07/18] Implement pack and unpack of big.Int --- db/rocksdb.go | 60 ++++++++++++++++++++++++++++++ db/rocksdb_test.go | 93 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 885b57d1..127b0cd8 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -6,6 +6,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" "os" "path/filepath" "time" @@ -1072,3 +1073,62 @@ func unpackVarint(buf []byte) (int32, int) { i, ofs := vlq.Int(buf) return int32(i), ofs } + +const ( + // number of bits in a big.Word + wordBits = 32 << (uint64(^big.Word(0)) >> 63) + // number of bytes in a big.Word + wordBytes = wordBits / 8 + // max packed bigint words + maxPackedBigintWords = (256 - wordBytes) / wordBytes +) + +// big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int +// number of written bytes is returned +// limitation: bigints longer than 248 bytes are truncated to 248 bytes +// caution: buffer must be big enough to hold the packed big int, buffer 249 bytes big is always safe +func packBigint(bi *big.Int, buf []byte) int { + w := bi.Bits() + lw := len(w) + // zero returns only one byte - zero length + if lw == 0 { + buf[0] = 0 + return 1 + } + // pack the most significant word in a special way - skip leading zeros + w0 := w[lw-1] + fb := 8 + mask := big.Word(0xff) << (wordBits - 8) + for w0&mask == 0 { + fb-- + mask >>= 8 + } + for i := fb; i > 0; i-- { + buf[i] = byte(w0) + w0 >>= 8 + } + // if the big int is too big (> 2^1984), the number of bytes would not fit to 1 byte + // in this case, truncate the number, it is not expected to work with this big numbers as amounts + s := 0 + if lw > maxPackedBigintWords { + s = lw - maxPackedBigintWords + } + // pack the rest of the words in reverse order + for j := lw - 2; j >= s; j-- { + d := w[j] + for i := fb + wordBytes; i > fb; i-- { + buf[i] = byte(d) + d >>= 8 + } + fb += wordBytes + } + buf[0] = byte(fb) + return fb + 1 +} + +func unpackBigint(buf []byte) (big.Int, int) { + var r big.Int + l := int(buf[0]) + 1 + r.SetBytes(buf[1:l]) + return r, l +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 52be1c5d..f34ecefb 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -1,4 +1,4 @@ -// +build unittest +// build unittest package db @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "os" "reflect" "sort" @@ -723,3 +724,93 @@ func Test_unpackBlockAddresses(t *testing.T) { }) } } + +func Test_packBigint_unpackBigint(t *testing.T) { + bigbig1, _ := big.NewInt(0).SetString("123456789123456789012345", 10) + bigbig2, _ := big.NewInt(0).SetString("12345678912345678901234512389012345123456789123456789012345123456789123456789012345", 10) + bigbigbig := big.NewInt(0) + bigbigbig.Mul(bigbig2, bigbig2) + bigbigbig.Mul(bigbigbig, bigbigbig) + bigbigbig.Mul(bigbigbig, bigbigbig) + tests := []struct { + name string + bi *big.Int + buf []byte + toobiglen int + }{ + { + name: "0", + bi: big.NewInt(0), + buf: make([]byte, 249), + }, + { + name: "1", + bi: big.NewInt(1), + buf: make([]byte, 249), + }, + { + name: "54321", + bi: big.NewInt(54321), + buf: make([]byte, 249), + }, + { + name: "12345678", + bi: big.NewInt(12345678), + buf: make([]byte, 249), + }, + { + name: "123456789123456789", + bi: big.NewInt(123456789123456789), + buf: make([]byte, 249), + }, + { + name: "bigbig1", + bi: bigbig1, + buf: make([]byte, 249), + }, + { + name: "bigbig2", + bi: bigbig2, + buf: make([]byte, 249), + }, + { + name: "bigbigbig", + bi: bigbigbig, + buf: make([]byte, 249), + toobiglen: 242, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // packBigint + got := packBigint(tt.bi, tt.buf) + if tt.toobiglen == 0 { + // create buffer that we expect + bb := tt.bi.Bytes() + want := append([]byte(nil), byte(len(bb))) + want = append(want, bb...) + if got != len(want) { + t.Errorf("packBigint() = %v, want %v", got, len(want)) + } + for i := 0; i < got; i++ { + if tt.buf[i] != want[i] { + t.Errorf("packBigint() buf = %v, want %v", tt.buf[:got], want) + break + } + } + // unpackBigint + got1, got2 := unpackBigint(tt.buf) + if got2 != len(want) { + t.Errorf("unpackBigint() = %v, want %v", got2, len(want)) + } + if tt.bi.Cmp(&got1) != 0 { + t.Errorf("unpackBigint() = %v, want %v", got1, tt.bi) + } + } else { + if got != tt.toobiglen { + t.Errorf("packBigint() = %v, want toobiglen %v", got, tt.toobiglen) + } + } + }) + } +} From a2bbf3f9de636dc0ce360b7b8fa3c7066807663d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 Aug 2018 14:30:45 +0200 Subject: [PATCH 08/18] Implement and test connectBlock for index v3 --- db/rocksdb.go | 450 ++++++++++++++++++++++++++++++++++++++++----- db/rocksdb_test.go | 332 +++++++++++++++++++-------------- 2 files changed, 603 insertions(+), 179 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 127b0cd8..89c920e5 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -22,7 +22,8 @@ import ( // when doing huge scan, it is better to close it and reopen from time to time to free the resources const refreshIterator = 5000000 const packedHeightBytes = 4 -const dbVersion = 0 +const dbVersion = 3 +const maxAddrIDLen = 1024 // RepairRocksDB calls RocksDb db repair function func RepairRocksDB(name string) error { @@ -203,11 +204,11 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f for _, o := range outpoints { var vout uint32 var isOutput bool - if o.vout < 0 { - vout = uint32(^o.vout) + if o.index < 0 { + vout = uint32(^o.index) isOutput = false } else { - vout = uint32(o.vout) + vout = uint32(o.index) isOutput = true } tx, err := d.chainParser.UnpackTxid(o.btxID) @@ -272,18 +273,381 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { type outpoint struct { btxID []byte - vout int32 + index int32 } +type txAddress struct { + addrID []byte + spent bool + valueSat big.Int +} + +type txAddresses struct { + inputs []txAddress + outputs []txAddress +} + +type addrBalance struct { + txs uint32 + sentSat big.Int + balanceSat big.Int +} + +func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } + addresses := make(map[string][]outpoint) + blockTxids := make([][]byte, len(block.Txs)) + txAddressesMap := make(map[string]*txAddresses) + balances := make(map[string]*addrBalance) + // first process all outputs so that inputs can point to txs in this block + for txi, tx := range block.Txs { + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return err + } + blockTxids[txi] = btxID + ta := txAddresses{} + ta.outputs = make([]txAddress, len(tx.Vout)) + txAddressesMap[string(btxID)] = &ta + for i, output := range tx.Vout { + tao := &ta.outputs[i] + tao.valueSat = output.ValueSat + addrID, err := d.chainParser.GetAddrIDFromVout(&output) + if err != nil || len(addrID) == 0 || len(addrID) > maxAddrIDLen { + if err != nil { + // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + } + } else { + glog.Infof("rocksdb: block %d, skipping addrID of length %d", block.Height, len(addrID)) + } + continue + } + tao.addrID = addrID + strAddrID := string(addrID) + // check that the address was used already in this block + o, processed := addresses[strAddrID] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, btxID) + } + addresses[strAddrID] = append(o, outpoint{ + btxID: btxID, + index: int32(i), + }) + ab, e := balances[strAddrID] + if !e { + ab, err = d.getAddressBalance(addrID) + if err != nil { + return err + } + if ab == nil { + ab = &addrBalance{} + } + balances[strAddrID] = ab + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.txs++ + } + ab.balanceSat.Add(&ab.balanceSat, &output.ValueSat) + } + } + // process inputs + for txi, tx := range block.Txs { + spendingTxid := blockTxids[txi] + ta := txAddressesMap[string(spendingTxid)] + ta.inputs = make([]txAddress, len(tx.Vin)) + for i, input := range tx.Vin { + tai := &ta.inputs[i] + btxID, err := d.chainParser.PackTxid(input.Txid) + if err != nil { + // do not process inputs without input txid + if err == bchain.ErrTxidMissing { + continue + } + return err + } + stxID := string(btxID) + ita, e := txAddressesMap[stxID] + if !e { + ita, err = d.getTxAddresses(btxID) + if err != nil { + return err + } + if ita == nil { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v not found in txAddresses", block.Height, tx.Txid, input.Txid) + continue + } + txAddressesMap[stxID] = ita + } + if len(ita.outputs) <= int(input.Vout) { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } + ot := &ita.outputs[int(input.Vout)] + if ot.spent { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } + tai.addrID = ot.addrID + tai.valueSat = ot.valueSat + // mark the output tx as spent + ot.spent = true + strAddrID := string(ot.addrID) + // check that the address was used already in this block + o, processed := addresses[strAddrID] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, spendingTxid) + } + addresses[strAddrID] = append(o, outpoint{ + btxID: spendingTxid, + index: ^int32(i), + }) + ab, e := balances[strAddrID] + if !e { + ab, err = d.getAddressBalance(ot.addrID) + if err != nil { + return err + } + if ab == nil { + ab = &addrBalance{} + } + balances[strAddrID] = ab + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.txs++ + } + ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) + if ab.balanceSat.Sign() < 0 { + ad, err := d.chainParser.OutputScriptToAddresses(ot.addrID) + if err != nil { + glog.Warningf("rocksdb: unparsable address reached negative balance %v, resetting to 0. Parser error %v", ab.balanceSat.String(), err) + } else { + glog.Warningf("rocksdb: address %v reached negative balance %v, resetting to 0", ab.balanceSat.String(), ad) + } + ab.balanceSat.SetInt64(0) + } + ab.sentSat.Add(&ab.sentSat, &ot.valueSat) + } + } + if op == opInsert { + if err := d.storeAddresses(wb, block, addresses); err != nil { + return err + } + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxids(wb, block, blockTxids); err != nil { + return err + } + } + return nil +} + +func processedInTx(o []outpoint, btxID []byte) bool { + for _, op := range o { + if bytes.Equal(btxID, op.btxID) { + return true + } + } + return false +} + +func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, block *bchain.Block, addresses map[string][]outpoint) error { + for addrID, outpoints := range addresses { + ba := []byte(addrID) + key := packAddressKey(ba, block.Height) + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + } + return nil +} + +func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAddresses) error { + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) + for txID, ta := range am { + buf = buf[:0] + l := packVaruint(uint(len(ta.inputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.inputs { + buf = appendTxAddress(buf, varBuf, &ta.inputs[i]) + } + l = packVaruint(uint(len(ta.outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.outputs { + buf = appendTxAddress(buf, varBuf, &ta.outputs[i]) + } + wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) + } + return nil +} + +func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBalance) error { + // allocate buffer big enough for number of txs + 2 bigints + buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) + for addrID, ab := range abm { + l := packVaruint(uint(ab.txs), buf) + ll := packBigint(&ab.sentSat, buf[l:]) + l += ll + ll = packBigint(&ab.balanceSat, buf[l:]) + l += ll + wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + } + return nil +} + +func appendTxAddress(buf []byte, varBuf []byte, txa *txAddress) []byte { + la := len(txa.addrID) + if txa.spent { + la = ^la + } + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txa.addrID...) + l = packBigint(&txa.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bchain.Block, txids [][]byte) error { + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, pl*len(txids)) + i := 0 + for _, txid := range txids { + copy(buf[i:], txid) + i += pl + } + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockTxids], key, buf) + keep := d.chainParser.KeepBlockAddresses() + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh < block.Height; rh-- { + key = packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], key) + if err != nil { + return err + } + if val.Size() == 0 { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxids], key) + } + } + return nil +} + +func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 3 is minimum length of addrBalance - 1 byte txs, 1 byte sent, 1 byte balance + if len(buf) < 3 { + return nil, nil + } + txs, l := unpackVaruint(buf) + sentSat, sl := unpackBigint(buf[l:]) + balanceSat, _ := unpackBigint(buf[l+sl:]) + return &addrBalance{ + txs: uint32(txs), + sentSat: sentSat, + balanceSat: balanceSat, + }, nil +} + +func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 2 is minimum length of addrBalance - 1 byte inputs len, 1 byte outputs len + if len(buf) < 2 { + return nil, nil + } + ta := txAddresses{} + inputs, l := unpackVaruint(buf) + ta.inputs = make([]txAddress, inputs) + for i := uint(0); i < inputs; i++ { + l += unpackTxAddress(&ta.inputs[i], buf[l:]) + } + outputs, ll := unpackVaruint(buf[l:]) + l += ll + ta.outputs = make([]txAddress, outputs) + for i := uint(0); i < outputs; i++ { + l += unpackTxAddress(&ta.outputs[i], buf[l:]) + } + return &ta, nil +} + +func unpackTxAddress(ta *txAddress, buf []byte) int { + al, l := unpackVarint(buf) + if al < 0 { + ta.spent = true + al ^= al + } + ta.addrID = make([]byte, al) + copy(ta.addrID, buf[l:l+al]) + al += l + ta.valueSat, l = unpackBigint(buf[al:]) + return l + al +} + +func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { + buf := make([]byte, 0) + bvout := make([]byte, vlq.MaxLen32) + for _, o := range outpoints { + l := packVarint32(o.index, bvout) + buf = append(buf, []byte(o.btxID)...) + buf = append(buf, bvout[:l]...) + } + return buf +} + +func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + outpoints := make([]outpoint, 0) + for i := 0; i < len(buf); { + btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) + i += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[i:]) + i += voutLen + outpoints = append(outpoints, outpoint{ + btxID: btxID, + index: vout, + }) + } + return outpoints, nil +} + +//////////////////////// + func (d *RocksDB) packBlockAddress(addrID []byte, spentTxs map[string][]outpoint) []byte { vBuf := make([]byte, vlq.MaxLen32) - vl := packVarint(int32(len(addrID)), vBuf) + vl := packVarint(len(addrID), vBuf) blockAddress := append([]byte(nil), vBuf[:vl]...) blockAddress = append(blockAddress, addrID...) if spentTxs == nil { } else { addrUnspentTxs := spentTxs[string(addrID)] - vl = packVarint(int32(len(addrUnspentTxs)), vBuf) + vl = packVarint(len(addrUnspentTxs), vBuf) blockAddress = append(blockAddress, vBuf[:vl]...) buf := d.packOutpoints(addrUnspentTxs) blockAddress = append(blockAddress, buf...) @@ -342,7 +706,7 @@ func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records m strAddrID := string(addrID) records[strAddrID] = append(records[strAddrID], outpoint{ btxID: btxid, - vout: vout, + index: vout, }) if op == opDelete { // remove transactions from cache @@ -370,10 +734,10 @@ func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) txAddrs = append(txAddrs, make([]byte, vlq.MaxLen32+len(addrID)+remaining*32)...)[:len(txAddrs)] } // addrID is packed as number of bytes of the addrID + bytes of addrID + vout - lv := packVarint(int32(len(addrID)), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) + lv := packVarint(len(addrID), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) txAddrs = txAddrs[:len(txAddrs)+lv] txAddrs = append(txAddrs, addrID...) - lv = packVarint(int32(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) + lv = packVarint(int(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) txAddrs = txAddrs[:len(txAddrs)+lv] return txAddrs } @@ -399,7 +763,7 @@ func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, []byte) return nil, unspentAddrs } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { +func (d *RocksDB) writeAddressesUTXO_old(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { if op == opDelete { // block does not contain mapping tx-> input address, which is necessary to recreate // unspentTxs; therefore it is not possible to DisconnectBlocks this way @@ -563,36 +927,9 @@ func (d *RocksDB) unpackBlockAddresses(buf []byte) ([][]byte, [][]outpoint, erro return addresses, outpointsArray, nil } -func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { - buf := make([]byte, 0) - bvout := make([]byte, vlq.MaxLen32) - for _, o := range outpoints { - l := packVarint(o.vout, bvout) - buf = append(buf, []byte(o.btxID)...) - buf = append(buf, bvout[:l]...) - } - return buf -} - -func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - outpoints := make([]outpoint, 0) - for i := 0; i < len(buf); { - btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) - i += txidUnpackedLen - vout, voutLen := unpackVarint(buf[i:]) - i += voutLen - outpoints = append(outpoints, outpoint{ - btxID: btxID, - vout: vout, - }) - } - return outpoints, nil -} - func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { txidUnpackedLen := d.chainParser.PackedTxidLen() - n, p := unpackVarint(buf) + n, p := unpackVarint32(buf) outpoints := make([]outpoint, n) for i := int32(0); i < n; i++ { if p+txidUnpackedLen >= len(buf) { @@ -600,11 +937,11 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { } btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) p += txidUnpackedLen - vout, voutLen := unpackVarint(buf[p:]) + vout, voutLen := unpackVarint32(buf[p:]) p += voutLen outpoints[i] = outpoint{ btxID: btxID, - vout: vout, + index: vout, } } return outpoints, p, nil @@ -616,7 +953,7 @@ func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { return nil, err } bv := make([]byte, vlq.MaxLen32) - l := packVarint(vout, bv) + l := packVarint32(vout, bv) buf := make([]byte, 0, l+len(btxid)) buf = append(buf, btxid...) buf = append(buf, bv[:l]...) @@ -626,10 +963,12 @@ func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { func (d *RocksDB) unpackOutpoint(buf []byte) (string, int32, int) { txidUnpackedLen := d.chainParser.PackedTxidLen() txid, _ := d.chainParser.UnpackTxid(buf[:txidUnpackedLen]) - vout, o := unpackVarint(buf[txidUnpackedLen:]) + vout, o := unpackVarint32(buf[txidUnpackedLen:]) return txid, vout, txidUnpackedLen + o } +////////////////////////////////// + // Block index // GetBestBlock returns the block hash of the block with highest height in the db @@ -799,7 +1138,7 @@ func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { return err } } - txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.vout), 1) + txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.index), 1) unspentTxs[stxID] = txAddrs } // delete unspentTxs from this block @@ -1065,15 +1404,33 @@ func unpackUint(buf []byte) uint32 { return binary.BigEndian.Uint32(buf) } -func packVarint(i int32, buf []byte) int { +func packVarint32(i int32, buf []byte) int { return vlq.PutInt(buf, int64(i)) } -func unpackVarint(buf []byte) (int32, int) { +func packVarint(i int, buf []byte) int { + return vlq.PutInt(buf, int64(i)) +} + +func packVaruint(i uint, buf []byte) int { + return vlq.PutUint(buf, uint64(i)) +} + +func unpackVarint32(buf []byte) (int32, int) { i, ofs := vlq.Int(buf) return int32(i), ofs } +func unpackVarint(buf []byte) (int, int) { + i, ofs := vlq.Int(buf) + return int(i), ofs +} + +func unpackVaruint(buf []byte) (uint, int) { + i, ofs := vlq.Uint(buf) + return uint(i), ofs +} + const ( // number of bits in a big.Word wordBits = 32 << (uint64(^big.Word(0)) >> 63) @@ -1081,6 +1438,7 @@ const ( wordBytes = wordBits / 8 // max packed bigint words maxPackedBigintWords = (256 - wordBytes) / wordBytes + maxPackedBigintBytes = 249 ) // big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index f34ecefb..75608bcc 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -60,6 +60,17 @@ func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string return strconv.FormatInt(int64(len(h)), 16) + h } +func spentAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return strconv.FormatInt(int64(len(h)+1), 16) + h +} + +func bigintToHex(i *big.Int) string { + b := make([]byte, maxPackedBigintBytes) + l := packBigint(i, b) + return hex.EncodeToString(b[:l]) +} + // keyPair is used to compare given key value in DB with expected // for more complicated compares it is possible to specify CompareFunc type keyPair struct { @@ -105,7 +116,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { valOK = kp[i].CompareFunc(val) } if !valOK { - return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, cfNames[col], i, kp[i].Value) + return errors.Errorf("Incorrect value %v found in column %v row %v key %v, expecting %v", val, cfNames[col], i, key, kp[i].Value) } i++ } @@ -115,6 +126,38 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } +const ( + txidB1T1 = "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + txidB1T2 = "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + txidB2T1 = "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + txidB2T2 = "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + txidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + + addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac + addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac + addr3 = "mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw" // 76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac + addr4 = "2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS" // a9144a21db08fb6882cb152e1ff06780a430740f770487 + addr5 = "2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1" // a914e921fc4912a315078f370d959f2c4f7b6d2a683c87 + addr6 = "mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX" // 76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac + addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac + addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac + addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac +) + +var ( + satZero = big.NewInt(0) + satB1T1A1 = big.NewInt(100000000) + satB1T1A2 = big.NewInt(12345) + satB1T2A3 = big.NewInt(1234567890123) + satB1T2A4 = big.NewInt(1) + satB1T2A5 = big.NewInt(9876) + satB2T1A6 = big.NewInt(317283951061) + satB2T1A7 = big.NewInt(917283951061) + satB2T2A8 = big.NewInt(118641975500) + satB2T2A9 = big.NewInt(198641975500) + satB2T3A5 = big.NewInt(9000) +) + func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ @@ -123,44 +166,49 @@ func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d), + Hex: addressToPubKeyHex(addr1, t, d), }, + ValueSat: *satB1T1A1, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d), + Hex: addressToPubKeyHex(addr2, t, d), }, + ValueSat: *satB1T1A2, }, }, Blocktime: 22549300000, Time: 22549300000, }, bchain.Tx{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d), + Hex: addressToPubKeyHex(addr3, t, d), }, + ValueSat: *satB1T2A3, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d), + Hex: addressToPubKeyHex(addr4, t, d), }, + ValueSat: *satB1T2A4, }, bchain.Vout{ N: 2, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB1T2A5, }, }, Blocktime: 22549300001, @@ -178,14 +226,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vin: []bchain.Vin{ + // addr3 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 0, }, + // addr2 bchain.Vin{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: 1, }, }, @@ -193,30 +243,32 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d), + Hex: addressToPubKeyHex(addr6, t, d), }, + ValueSat: *satB2T1A6, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d), + Hex: addressToPubKeyHex(addr7, t, d), }, + ValueSat: *satB2T1A7, }, }, Blocktime: 22549400000, Time: 22549400000, }, bchain.Tx{ - Txid: "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", + Txid: txidB2T2, Vin: []bchain.Vin{ - // spending an output in the same block + // spending an output in the same block - addr6 bchain.Vin{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vout: 0, }, - // spending an output in the previous block + // spending an output in the previous block - addr4 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 1, }, }, @@ -224,14 +276,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d), + Hex: addressToPubKeyHex(addr8, t, d), }, + ValueSat: *satB2T2A8, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d), + Hex: addressToPubKeyHex(addr9, t, d), }, + ValueSat: *satB2T2A9, }, }, Blocktime: 22549400001, @@ -239,10 +293,11 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, // transaction from the same address in the previous block bchain.Tx{ - Txid: "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", + Txid: txidB2T3, Vin: []bchain.Vin{ + // addr5 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 2, }, }, @@ -250,8 +305,9 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB2T3A5, }, }, Blocktime: 22549400002, @@ -261,7 +317,7 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { } } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, }); err != nil { @@ -271,62 +327,51 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { } // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", - }) - }, + txidB1T1, + "00" + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), + nil, }, keyPair{ - "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "04", - }) - }, + txidB1T2, + "00" + "03" + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), + nil, }, }); err != nil { { t.Fatal(err) } } - // after disconnect there are no blockaddresses for the previous block - var blockAddressesKp []keyPair - if noBlockAddresses { - blockAddressesKp = []keyPair{} - } else { - // the values in cfBlockAddresses are in random order, must use CompareFunc - blockAddressesKp = []keyPair{ - keyPair{"000370d5", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "00", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", - }) - }, - }, + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A2), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A3), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A4), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A5), nil}, + }); err != nil { + { + t.Fatal(err) } } - if err := checkColumn(d, cfBlockAddresses, blockAddressesKp); err != nil { + if err := checkColumn(d, cfBlockTxids, []keyPair{ + keyPair{"000370d5", txidB1T1 + txidB1T2, nil}, + }); err != nil { { t.Fatal(err) } @@ -343,47 +388,66 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { } } if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, - keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil}, - keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil}, - keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil}, - keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d6", "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "00" + "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "01", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, + keyPair{addressToPubKeyHex(addr6, t, d) + "000370d6", txidB2T1 + "00" + txidB2T2 + "01", nil}, + keyPair{addressToPubKeyHex(addr7, t, d) + "000370d6", txidB2T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr8, t, d) + "000370d6", txidB2T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr9, t, d) + "000370d6", txidB2T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d6", txidB2T1 + "01", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d6", txidB2T1 + "03", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d6", txidB2T3 + "00" + txidB2T3 + "01", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d6", txidB2T2 + "03", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + txidB1T1, + "00" + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + spentAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), nil, }, keyPair{ - "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + txidB1T2, + "00" + "03" + + spentAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + spentAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + spentAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), nil, }, keyPair{ - "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", - }) - }, + txidB2T1, + "02" + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + + "02" + + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), + nil, }, keyPair{ - "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", + txidB2T2, + "02" + + addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + "02" + + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), + nil, + }, + keyPair{ + txidB2T3, + "01" + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + + "01" + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, }, }); err != nil { @@ -391,21 +455,23 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockAddresses, []keyPair{ - keyPair{"000370d6", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "00", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "00", - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02" + "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", - }) - }, - }, + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "02" + bigintToHex(satB1T1A2) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "02" + bigintToHex(satB1T2A3) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "02" + bigintToHex(satB1T2A4) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "02" + bigintToHex(satB1T2A5) + bigintToHex(satB2T3A5), nil}, + keyPair{addressToPubKeyHex(addr6, t, d), "02" + bigintToHex(satB2T1A6) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr7, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T1A7), nil}, + keyPair{addressToPubKeyHex(addr8, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A8), nil}, + keyPair{addressToPubKeyHex(addr9, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A9), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfBlockTxids, []keyPair{ + keyPair{"000370d6", txidB2T1 + txidB2T2 + txidB2T3, nil}, }); err != nil { { t.Fatal(err) @@ -487,12 +553,12 @@ func TestRocksDB_Index_UTXO(t *testing.T) { }) defer closeAndDestroyRocksDB(t, d) - // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column + // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column block1 := getTestUTXOBlock1(t, d) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, false) + verifyAfterUTXOBlock1(t, d) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block block2 := getTestUTXOBlock2(t, d) @@ -502,19 +568,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { verifyAfterUTXOBlock2(t, d) // get transactions for various addresses / low-high ranges - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225493, 225493, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, + verifyGetTransactions(t, d, addr2, 225493, 225493, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225494, 1000000, []txidVoutOutput{ - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 225494, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 500000, 1000000, []txidVoutOutput{}, nil) - verifyGetTransactions(t, d, "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", 0, true}, + verifyGetTransactions(t, d, addr2, 500000, 1000000, []txidVoutOutput{}, nil) + verifyGetTransactions(t, d, addr8, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T2, 0, true}, }, nil) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch")) @@ -569,7 +635,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, true) + verifyAfterUTXOBlock1(t, d) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { t.Fatal(err) @@ -661,13 +727,13 @@ func Test_unpackBlockAddresses(t *testing.T) { want2: [][]hexoutpoint{ []hexoutpoint{}, []hexoutpoint{ - hexoutpoint{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 0}, - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 3}, + hexoutpoint{txidB2T1, 0}, + hexoutpoint{txidB1T1, 3}, }, []hexoutpoint{}, []hexoutpoint{}, []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + hexoutpoint{txidB1T2, 1}, }, }, }, @@ -679,13 +745,13 @@ func Test_unpackBlockAddresses(t *testing.T) { []hexoutpoint{}, []hexoutpoint{}, []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 0}, + hexoutpoint{txidB1T2, 0}, }, []hexoutpoint{ - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1}, + hexoutpoint{txidB1T1, 1}, }, []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + hexoutpoint{txidB1T2, 1}, }, []hexoutpoint{}, []hexoutpoint{}, @@ -714,7 +780,7 @@ func Test_unpackBlockAddresses(t *testing.T) { for i, g := range got2 { ho := make([]hexoutpoint, len(g)) for j, o := range g { - ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.vout} + ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.index} } h2[i] = ho } @@ -741,12 +807,12 @@ func Test_packBigint_unpackBigint(t *testing.T) { { name: "0", bi: big.NewInt(0), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "1", bi: big.NewInt(1), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "54321", @@ -756,27 +822,27 @@ func Test_packBigint_unpackBigint(t *testing.T) { { name: "12345678", bi: big.NewInt(12345678), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "123456789123456789", bi: big.NewInt(123456789123456789), - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "bigbig1", bi: bigbig1, - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "bigbig2", bi: bigbig2, - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), }, { name: "bigbigbig", bi: bigbigbig, - buf: make([]byte, 249), + buf: make([]byte, maxPackedBigintBytes), toobiglen: 242, }, } From fb93c9ff7a825ece512ac6ebcaac8247becd0970 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 Aug 2018 16:10:28 +0200 Subject: [PATCH 09/18] Add unit test for pack/unpack txAddress --- db/rocksdb.go | 59 ++++++++++-------- db/rocksdb_test.go | 148 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 178 insertions(+), 29 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 89c920e5..dc33debb 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -478,17 +478,7 @@ func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAd varBuf := make([]byte, maxPackedBigintBytes) buf := make([]byte, 1024) for txID, ta := range am { - buf = buf[:0] - l := packVaruint(uint(len(ta.inputs)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ta.inputs { - buf = appendTxAddress(buf, varBuf, &ta.inputs[i]) - } - l = packVaruint(uint(len(ta.outputs)), varBuf) - buf = append(buf, varBuf[:l]...) - for i := range ta.outputs { - buf = appendTxAddress(buf, varBuf, &ta.outputs[i]) - } + buf = packTxAddresses(ta, buf, varBuf) wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) } return nil @@ -508,19 +498,6 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa return nil } -func appendTxAddress(buf []byte, varBuf []byte, txa *txAddress) []byte { - la := len(txa.addrID) - if txa.spent { - la = ^la - } - l := packVarint(la, varBuf) - buf = append(buf, varBuf[:l]...) - buf = append(buf, txa.addrID...) - l = packBigint(&txa.valueSat, varBuf) - buf = append(buf, varBuf[:l]...) - return buf -} - func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bchain.Block, txids [][]byte) error { pl := d.chainParser.PackedTxidLen() buf := make([]byte, pl*len(txids)) @@ -582,6 +559,38 @@ func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { if len(buf) < 2 { return nil, nil } + return unpackTxAddresses(buf) +} + +func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { + buf = buf[:0] + l := packVaruint(uint(len(ta.inputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.inputs { + buf = appendTxAddress(&ta.inputs[i], buf, varBuf) + } + l = packVaruint(uint(len(ta.outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.outputs { + buf = appendTxAddress(&ta.outputs[i], buf, varBuf) + } + return buf +} + +func appendTxAddress(txa *txAddress, buf []byte, varBuf []byte) []byte { + la := len(txa.addrID) + if txa.spent { + la = ^la + } + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txa.addrID...) + l = packBigint(&txa.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func unpackTxAddresses(buf []byte) (*txAddresses, error) { ta := txAddresses{} inputs, l := unpackVaruint(buf) ta.inputs = make([]txAddress, inputs) @@ -601,7 +610,7 @@ func unpackTxAddress(ta *txAddress, buf []byte) int { al, l := unpackVarint(buf) if al < 0 { ta.spent = true - al ^= al + al = ^al } ta.addrID = make([]byte, al) copy(ta.addrID, buf[l:l+al]) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 75608bcc..52aa3b29 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -23,6 +23,13 @@ import ( // for number n, the packing is: 2*n if n>=0 else 2*(-n)-1 // takes only 1 byte if abs(n)<127 +func bitcoinTestnetParser() *btc.BitcoinParser { + return &btc.BitcoinParser{ + BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, + Params: btc.GetChainParams("test"), + } +} + func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { tmp, err := ioutil.TempDir("", "testdb") if err != nil { @@ -546,10 +553,7 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ - BitcoinParser: &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, - Params: btc.GetChainParams("test"), - }, + BitcoinParser: bitcoinTestnetParser(), }) defer closeAndDestroyRocksDB(t, d) @@ -880,3 +884,139 @@ func Test_packBigint_unpackBigint(t *testing.T) { }) } } + +func addressToOutput(addr string, parser *btc.BitcoinParser) []byte { + b, err := parser.AddressToOutputScript(addr) + if err != nil { + panic(err) + } + return b +} + +func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { + parser := bitcoinTestnetParser() + tests := []struct { + name string + hex string + data *txAddresses + }{ + { + name: "1", + hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b7002c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + data: &txAddresses{ + inputs: []txAddress{ + { + addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + valueSat: *big.NewInt(0), + }, + { + addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), + valueSat: *big.NewInt(1234123421342341234), + }, + }, + outputs: []txAddress{ + { + addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + valueSat: *big.NewInt(1), + spent: true, + }, + }, + }, + }, + { + name: "2", + hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c02ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c02ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + data: &txAddresses{ + inputs: []txAddress{ + { + addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + valueSat: *big.NewInt(9011000000), + }, + { + addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + valueSat: *big.NewInt(8011000000), + }, + { + addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + valueSat: *big.NewInt(7011000000), + }, + }, + outputs: []txAddress{ + { + addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + valueSat: *big.NewInt(5011000000), + spent: true, + }, + { + addrID: addressToOutput("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + valueSat: *big.NewInt(6011000000), + }, + { + addrID: addressToOutput("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + valueSat: *big.NewInt(7011000000), + spent: true, + }, + { + addrID: addressToOutput("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + valueSat: *big.NewInt(999900000), + }, + { + addrID: addressToOutput("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + valueSat: *big.NewInt(5000000000), + spent: true, + }, + }, + }, + }, + { + name: "empty address", + hex: "01000204d2020002162e010162", + data: &txAddresses{ + inputs: []txAddress{ + { + addrID: []byte{}, + valueSat: *big.NewInt(1234), + }, + }, + outputs: []txAddress{ + { + addrID: []byte{}, + valueSat: *big.NewInt(5678), + }, + { + addrID: []byte{}, + valueSat: *big.NewInt(98), + spent: true, + }, + }, + }, + }, + { + name: "empty", + hex: "0000", + data: &txAddresses{ + inputs: []txAddress{}, + outputs: []txAddress{}, + }, + }, + } + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := packTxAddresses(tt.data, buf, varBuf) + hex := hex.EncodeToString(b) + if !reflect.DeepEqual(hex, tt.hex) { + t.Errorf("packTxAddresses() = %v, want %v", hex, tt.hex) + } + got1, err := unpackTxAddresses(b) + if err != nil { + t.Errorf("unpackTxAddresses() error = %v", err) + return + } + if !reflect.DeepEqual(got1, tt.data) { + t.Errorf("unpackTxAddresses() = %+v, want %+v", got1, tt.data) + } + }) + } +} From a04e38f0f7bab2d41223534bd4bdabe43f326f3a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 2 Aug 2018 17:46:23 +0200 Subject: [PATCH 10/18] Improve handling of invalid addresses --- db/rocksdb.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index dc33debb..e71f79d7 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -324,7 +324,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) } } else { - glog.Infof("rocksdb: block %d, skipping addrID of length %d", block.Height, len(addrID)) + glog.Infof("rocksdb: height %d, tx %v, vout %v, skipping addrID of length %d", block.Height, tx.Txid, i, len(addrID)) } continue } @@ -397,8 +397,12 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } tai.addrID = ot.addrID tai.valueSat = ot.valueSat - // mark the output tx as spent + // mark the output as spent in tx ot.spent = true + if len(ot.addrID) == 0 { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } strAddrID := string(ot.addrID) // check that the address was used already in this block o, processed := addresses[strAddrID] @@ -428,10 +432,11 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) if ab.balanceSat.Sign() < 0 { ad, err := d.chainParser.OutputScriptToAddresses(ot.addrID) + had := hex.EncodeToString(ot.addrID) if err != nil { - glog.Warningf("rocksdb: unparsable address reached negative balance %v, resetting to 0. Parser error %v", ab.balanceSat.String(), err) + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative balance %v, resetting to 0. Parser error %v", had, ab.balanceSat.String(), err) } else { - glog.Warningf("rocksdb: address %v reached negative balance %v, resetting to 0", ab.balanceSat.String(), ad) + glog.Warningf("rocksdb: address %v hex '%v' reached negative balance %v, resetting to 0", ad, had, ab.balanceSat.String()) } ab.balanceSat.SetInt64(0) } From 7e11a4e615ff947e7fcef5b5567eedec9ee44196 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 3 Aug 2018 15:04:49 +0200 Subject: [PATCH 11/18] Set LZ4HC compression for all columns --- db/rocksdb.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index e71f79d7..ebee699b 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -88,7 +88,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) optsLZ4.SetBytesPerSync(1 << 20) // 1MB optsLZ4.SetWriteBufferSize(1 << 27) // 128MB optsLZ4.SetMaxOpenFiles(25000) - optsLZ4.SetCompression(gorocksdb.LZ4Compression) + optsLZ4.SetCompression(gorocksdb.LZ4HCCompression) // opts for addresses are different: // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter @@ -105,10 +105,10 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) optsAddresses.SetBytesPerSync(1 << 20) // 1MB optsAddresses.SetWriteBufferSize(1 << 27) // 128MB optsAddresses.SetMaxOpenFiles(25000) - optsAddresses.SetCompression(gorocksdb.NoCompression) + optsAddresses.SetCompression(gorocksdb.LZ4HCCompression) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions - fcOptions := []*gorocksdb.Options{optsNoCompression, optsNoCompression, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} + fcOptions := []*gorocksdb.Options{optsLZ4, optsLZ4, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions) if err != nil { From 8e3c7f851bb7770de9aa9e7dbe5949b32d98c670 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 3 Aug 2018 19:26:16 +0200 Subject: [PATCH 12/18] Clean up the rocksdb sources and disconnect UTXO blocks WIP --- db/rocksdb.go | 517 ++++++++++++--------------------------------- db/rocksdb_test.go | 156 +------------- db/sync.go | 14 +- 3 files changed, 148 insertions(+), 539 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index ebee699b..7ce84b54 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -52,10 +52,6 @@ const ( cfAddressBalance cfBlockTxids cfTransactions - - // to be removed, kept temporarily so that the code is compilable - cfUnspentTxs - cfBlockAddresses ) var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"} @@ -257,7 +253,12 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return err } if isUTXO { - if err := d.writeAddressesUTXO(wb, block, op); err != nil { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } + if err := d.writeAddressesUTXO(wb, block); err != nil { return err } } else { @@ -293,12 +294,7 @@ type addrBalance struct { balanceSat big.Int } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } +func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { addresses := make(map[string][]outpoint) blockTxids := make([][]byte, len(block.Txs)) txAddressesMap := make(map[string]*txAddresses) @@ -443,21 +439,16 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } } - if op == opInsert { - if err := d.storeAddresses(wb, block, addresses); err != nil { - return err - } - if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { - return err - } - if err := d.storeBalances(wb, balances); err != nil { - return err - } - if err := d.storeAndCleanupBlockTxids(wb, block, blockTxids); err != nil { - return err - } + if err := d.storeAddresses(wb, block, addresses); err != nil { + return err } - return nil + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + return d.storeAndCleanupBlockTxids(wb, block, blockTxids) } func processedInTx(o []outpoint, btxID []byte) bool { @@ -493,12 +484,17 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa // allocate buffer big enough for number of txs + 2 bigints buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) for addrID, ab := range abm { - l := packVaruint(uint(ab.txs), buf) - ll := packBigint(&ab.sentSat, buf[l:]) - l += ll - ll = packBigint(&ab.balanceSat, buf[l:]) - l += ll - wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + // balance with 0 transactions is removed from db - happens in disconnect + if ab == nil || ab.txs <= 0 { + wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrID)) + } else { + l := packVaruint(uint(ab.txs), buf) + ll := packBigint(&ab.sentSat, buf[l:]) + l += ll + ll = packBigint(&ab.balanceSat, buf[l:]) + l += ll + wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + } } return nil } @@ -532,6 +528,22 @@ func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bch return nil } +func (d *RocksDB) getBlockTxids(height uint32) ([][]byte, error) { + pl := d.chainParser.PackedTxidLen() + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], packUint(height)) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + txids := make([][]byte, len(buf)/pl) + for i := 0; i < len(txids); i++ { + txid := make([]byte, pl) + copy(txid, buf[pl*i:]) + } + return txids, nil +} + func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) if err != nil { @@ -651,70 +663,9 @@ func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { return outpoints, nil } -//////////////////////// - -func (d *RocksDB) packBlockAddress(addrID []byte, spentTxs map[string][]outpoint) []byte { - vBuf := make([]byte, vlq.MaxLen32) - vl := packVarint(len(addrID), vBuf) - blockAddress := append([]byte(nil), vBuf[:vl]...) - blockAddress = append(blockAddress, addrID...) - if spentTxs == nil { - } else { - addrUnspentTxs := spentTxs[string(addrID)] - vl = packVarint(len(addrUnspentTxs), vBuf) - blockAddress = append(blockAddress, vBuf[:vl]...) - buf := d.packOutpoints(addrUnspentTxs) - blockAddress = append(blockAddress, buf...) - } - return blockAddress -} - -func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, addresses map[string][]outpoint, spentTxs map[string][]outpoint) error { - keep := d.chainParser.KeepBlockAddresses() - blockAddresses := make([]byte, 0) - for addrID, outpoints := range addresses { - baddrID := []byte(addrID) - key := packAddressKey(baddrID, block.Height) - switch op { - case opInsert: - val := d.packOutpoints(outpoints) - wb.PutCF(d.cfh[cfAddresses], key, val) - if keep > 0 { - // collect all addresses be stored in blockaddresses - // they are used in disconnect blocks - blockAddress := d.packBlockAddress(baddrID, spentTxs) - blockAddresses = append(blockAddresses, blockAddress...) - } - case opDelete: - wb.DeleteCF(d.cfh[cfAddresses], key) - } - } - if keep > 0 && op == opInsert { - // write new block address and txs spent in this block - key := packUint(block.Height) - wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses) - // cleanup old block address - if block.Height > uint32(keep) { - for rh := block.Height - uint32(keep); rh < block.Height; rh-- { - key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return err - } - if val.Size() == 0 { - break - } - val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key) - } - } - } - return nil -} - func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { if len(addrID) > 0 { - if len(addrID) > 1024 { + if len(addrID) > maxAddrIDLen { glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) } else { strAddrID := string(addrID) @@ -731,153 +682,6 @@ func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records m return nil } -func (d *RocksDB) getUnspentTx(btxID []byte) ([]byte, error) { - // find it in db, in the column cfUnspentTxs - val, err := d.db.GetCF(d.ro, d.cfh[cfUnspentTxs], btxID) - if err != nil { - return nil, err - } - defer val.Free() - data := append([]byte(nil), val.Data()...) - return data, nil -} - -func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) []byte { - // resize the addr buffer if necessary by a new estimate - if cap(txAddrs)-len(txAddrs) < 2*vlq.MaxLen32+len(addrID) { - txAddrs = append(txAddrs, make([]byte, vlq.MaxLen32+len(addrID)+remaining*32)...)[:len(txAddrs)] - } - // addrID is packed as number of bytes of the addrID + bytes of addrID + vout - lv := packVarint(len(addrID), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - txAddrs = append(txAddrs, addrID...) - lv = packVarint(int(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - return txAddrs -} - -func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, []byte) { - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(unspentAddrs); { - l, lv1 := unpackVarint(unspentAddrs[i:]) - // index of vout of address in unspentAddrs - j := i + int(l) + lv1 - if j >= len(unspentAddrs) { - glog.Error("rocksdb: Inconsistent data in unspentAddrs ", hex.EncodeToString(unspentAddrs), ", ", vout) - return nil, unspentAddrs - } - n, lv2 := unpackVarint(unspentAddrs[j:]) - if uint32(n) == vout { - addrID := append([]byte(nil), unspentAddrs[i+lv1:j]...) - unspentAddrs = append(unspentAddrs[:i], unspentAddrs[j+lv2:]...) - return addrID, unspentAddrs - } - i = j + lv2 - } - return nil, unspentAddrs -} - -func (d *RocksDB) writeAddressesUTXO_old(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } - addresses := make(map[string][]outpoint) - unspentTxs := make(map[string][]byte) - thisBlockTxs := make(map[string]struct{}) - btxIDs := make([][]byte, len(block.Txs)) - // first process all outputs, build mapping of addresses to outpoints and mappings of unspent txs to addresses - for txi, tx := range block.Txs { - btxID, err := d.chainParser.PackTxid(tx.Txid) - if err != nil { - return err - } - btxIDs[txi] = btxID - // preallocate estimated size of addresses (32 bytes is 1 byte length of addrID, 25 bytes addrID, 1-2 bytes vout and reserve) - txAddrs := make([]byte, 0, len(tx.Vout)*32) - for i, output := range tx.Vout { - addrID, err := d.chainParser.GetAddrIDFromVout(&output) - if err != nil { - // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) - if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) - } - continue - } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(output.N), block.Height) - if err != nil { - return err - } - txAddrs = appendPackedAddrID(txAddrs, addrID, output.N, len(tx.Vout)-i) - } - stxID := string(btxID) - unspentTxs[stxID] = txAddrs - thisBlockTxs[stxID] = struct{}{} - } - // locate addresses spent by this tx and remove them from unspent addresses - // keep them so that they be stored for DisconnectBlock functionality - spentTxs := make(map[string][]outpoint) - for txi, tx := range block.Txs { - spendingTxid := btxIDs[txi] - for i, input := range tx.Vin { - btxID, err := d.chainParser.PackTxid(input.Txid) - if err != nil { - // do not process inputs without input txid - if err == bchain.ErrTxidMissing { - continue - } - return err - } - // find the tx in current block or already processed - stxID := string(btxID) - unspentAddrs, exists := unspentTxs[stxID] - if !exists { - // else find it in previous blocks - unspentAddrs, err = d.getUnspentTx(btxID) - if err != nil { - return err - } - if unspentAddrs == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v missing in unspentTxs", block.Height, tx.Txid, input.Txid, input.Vout, i) - continue - } - } - var addrID []byte - addrID, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout) - if addrID == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v not found in unspentAddrs", block.Height, tx.Txid, input.Txid, input.Vout, i) - continue - } - // record what was spent in this tx - // skip transactions that were created in this block - if _, exists := thisBlockTxs[stxID]; !exists { - saddrID := string(addrID) - rut := spentTxs[saddrID] - rut = append(rut, outpoint{btxID, int32(input.Vout)}) - spentTxs[saddrID] = rut - } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, spendingTxid, int32(^i), block.Height) - if err != nil { - return err - } - unspentTxs[stxID] = unspentAddrs - } - } - if err := d.writeAddressRecords(wb, block, op, addresses, spentTxs); err != nil { - return err - } - // save unspent txs from current block - for tx, val := range unspentTxs { - if len(val) == 0 { - wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) - } else { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) - } - } - return nil -} - func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { addresses := make(map[string][]outpoint) for _, tx := range block.Txs { @@ -914,75 +718,19 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. } } } - return d.writeAddressRecords(wb, block, op, addresses, nil) -} - -func (d *RocksDB) unpackBlockAddresses(buf []byte) ([][]byte, [][]outpoint, error) { - addresses := make([][]byte, 0) - outpointsArray := make([][]outpoint, 0) - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(buf); { - l, lv := unpackVarint(buf[i:]) - j := i + int(l) + lv - if j > len(buf) { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addrID := append([]byte(nil), buf[i+lv:j]...) - outpoints, ol, err := d.unpackNOutpoints(buf[j:]) - if err != nil { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addresses = append(addresses, addrID) - outpointsArray = append(outpointsArray, outpoints) - i = j + ol - } - return addresses, outpointsArray, nil -} - -func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - n, p := unpackVarint32(buf) - outpoints := make([]outpoint, n) - for i := int32(0); i < n; i++ { - if p+txidUnpackedLen >= len(buf) { - return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") - } - btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) - p += txidUnpackedLen - vout, voutLen := unpackVarint32(buf[p:]) - p += voutLen - outpoints[i] = outpoint{ - btxID: btxID, - index: vout, + for addrID, outpoints := range addresses { + key := packAddressKey([]byte(addrID), block.Height) + switch op { + case opInsert: + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + case opDelete: + wb.DeleteCF(d.cfh[cfAddresses], key) } } - return outpoints, p, nil + return nil } -func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { - btxid, err := d.chainParser.PackTxid(txid) - if err != nil { - return nil, err - } - bv := make([]byte, vlq.MaxLen32) - l := packVarint32(vout, bv) - buf := make([]byte, 0, l+len(btxid)) - buf = append(buf, btxid...) - buf = append(buf, bv[:l]...) - return buf, nil -} - -func (d *RocksDB) unpackOutpoint(buf []byte) (string, int32, int) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - txid, _ := d.chainParser.UnpackTxid(buf[:txidUnpackedLen]) - vout, o := unpackVarint32(buf[txidUnpackedLen:]) - return txid, vout, txidUnpackedLen + o -} - -////////////////////////////////// - // Block index // GetBestBlock returns the block hash of the block with highest height in the db @@ -1032,18 +780,7 @@ func (d *RocksDB) writeHeight( return nil } -func (d *RocksDB) getBlockAddresses(key []byte) ([][]byte, [][]outpoint, error) { - b, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return nil, nil, err - } - defer b.Free() - // block is missing in DB - if b.Data() == nil { - return nil, nil, errors.New("Block addresses missing") - } - return d.unpackBlockAddresses(b.Data()) -} +// Disconnect blocks func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]byte, error) { glog.Infof("db: doing full scan of addresses column") @@ -1090,93 +827,107 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -// DisconnectBlockRange removes all data belonging to blocks in range lower-higher -// it finds the data in blockaddresses column if available, -// otherwise by doing quite slow full scan of addresses column -func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys := [][]byte{} - addrOutpoints := [][]byte{} - addrUnspentOutpoints := [][]outpoint{} - keep := d.chainParser.KeepBlockAddresses() - var err error - if keep > 0 { - for height := lower; height <= higher; height++ { - addresses, unspentOutpoints, err := d.getBlockAddresses(packUint(height)) +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, txAddressesToUpdate map[string]*txAddresses, txsToDelete map[string]struct{}, balances map[string]*addrBalance) error { + findBalance := func(addrID []byte) (*addrBalance, error) { + var err error + s := string(addrID) + b, found := balances[s] + if !found { + b, err = d.getAddressBalance(addrID) if err != nil { - glog.Error(err) - return err - } - for i, addrID := range addresses { - addrKey := packAddressKey(addrID, height) - val, err := d.db.GetCF(d.ro, d.cfh[cfAddresses], addrKey) - if err != nil { - glog.Error(err) - return err - } - addrKeys = append(addrKeys, addrKey) - av := append([]byte(nil), val.Data()...) - val.Free() - addrOutpoints = append(addrOutpoints, av) - addrUnspentOutpoints = append(addrUnspentOutpoints, unspentOutpoints[i]) + return nil, err } + balances[s] = b } - } else { - addrKeys, addrOutpoints, err = d.allAddressesScan(lower, higher) + return b, nil + } + for _, t := range txa.inputs { + b, err := findBalance(t.addrID) if err != nil { return err } + if b != nil { + + } } + return nil +} + +// DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher +// if they are in the range kept in the cfBlockTxs column +func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + blocksTxids := make([][][]byte, higher-lower+1) + for height := lower; height <= higher; height++ { + blockTxids, err := d.getBlockTxids(height) + if err != nil { + return err + } + if len(blockTxids) == 0 { + return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) + } + blocksTxids[height-lower] = blockTxids + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + txAddressesToUpdate := make(map[string]*txAddresses) + txsToDelete := make(map[string]struct{}) + balances := make(map[string]*addrBalance) + for height := higher; height >= lower; height-- { + blockTxids := blocksTxids[height-lower] + for _, txid := range blockTxids { + txa, err := d.getTxAddresses(txid) + if err != nil { + return err + } + s := string(txid) + txsToDelete[s] = struct{}{} + if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, txsToDelete, balances); err != nil { + return err + } + } + key := packUint(height) + wb.DeleteCF(d.cfh[cfBlockTxids], key) + wb.DeleteCF(d.cfh[cfHeight], key) + } + d.storeTxAddresses(wb, txAddressesToUpdate) + d.storeBalances(wb, balances) + for s := range txsToDelete { + b := []byte(s) + wb.DeleteCF(d.cfh[cfTransactions], b) + wb.DeleteCF(d.cfh[cfTxAddresses], b) + } + err := d.db.Write(d.wo, wb) + if err == nil { + glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) + } + return err +} + +// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks +// it is very slow operation +func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + addrKeys, _, err := d.allAddressesScan(lower, higher) + if err != nil { + return err + } glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - unspentTxs := make(map[string][]byte) - for addrIndex, addrKey := range addrKeys { + for _, addrKey := range addrKeys { if glog.V(2) { glog.Info("address ", hex.EncodeToString(addrKey)) } // delete address:height from the index wb.DeleteCF(d.cfh[cfAddresses], addrKey) - addrID, _, err := unpackAddressKey(addrKey) - if err != nil { - return err - } - // recreate unspentTxs, which were spent by this block (that is being disconnected) - for _, o := range addrUnspentOutpoints[addrIndex] { - stxID := string(o.btxID) - txAddrs, exists := unspentTxs[stxID] - if !exists { - txAddrs, err = d.getUnspentTx(o.btxID) - if err != nil { - return err - } - } - txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.index), 1) - unspentTxs[stxID] = txAddrs - } - // delete unspentTxs from this block - outpoints, err := d.unpackOutpoints(addrOutpoints[addrIndex]) - if err != nil { - return err - } - for _, o := range outpoints { - wb.DeleteCF(d.cfh[cfUnspentTxs], o.btxID) - d.internalDeleteTx(wb, o.btxID) - } - } - for key, val := range unspentTxs { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(key), val) } for height := lower; height <= higher; height++ { if glog.V(2) { glog.Info("height ", height) } - key := packUint(height) - if keep > 0 { - wb.DeleteCF(d.cfh[cfBlockAddresses], key) - } - wb.DeleteCF(d.cfh[cfHeight], key) + wb.DeleteCF(d.cfh[cfHeight], packUint(height)) } err = d.db.Write(d.wo, wb) if err == nil { diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 52aa3b29..4a96e769 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -632,9 +632,16 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } verifyAfterUTXOBlock2(t, d) + // try to disconnect both blocks, however only the last one is kept, it is not possible + err = d.DisconnectBlockRangeUTXO(225493, 225494) + if err == nil || err.Error() != "Cannot disconnect blocks with height 225493 and lower. It is necessary to rebuild index." { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs // and that the cached tx is removed - err = d.DisconnectBlockRange(225494, 225494) + err = d.DisconnectBlockRangeUTXO(225494, 225494) if err != nil { t.Fatal(err) } @@ -648,153 +655,6 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } -func Test_findAndRemoveUnspentAddr(t *testing.T) { - type args struct { - unspentAddrs string - vout uint32 - } - tests := []struct { - name string - args args - want string - want2 string - }{ - { - name: "3", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 3, - }, - want: "64635167006868", - want2: "029c0010517a0115887452870212709393588893935687040e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - }, - { - name: "10", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 10, - }, - want: "61", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112", - }, - { - name: "not there", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 11, - }, - want: "", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.unspentAddrs) - if err != nil { - panic(err) - } - got, got2 := findAndRemoveUnspentAddr(b, tt.args.vout) - h := hex.EncodeToString(got) - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("findAndRemoveUnspentAddr() got = %v, want %v", h, tt.want) - } - h2 := hex.EncodeToString(got2) - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("findAndRemoveUnspentAddr() got2 = %v, want %v", h2, tt.want2) - } - }) - } -} - -type hexoutpoint struct { - txID string - vout int32 -} - -func Test_unpackBlockAddresses(t *testing.T) { - d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}}) - defer closeAndDestroyRocksDB(t, d) - type args struct { - buf string - } - tests := []struct { - name string - args args - want []string - want2 [][]hexoutpoint - wantErr bool - }{ - { - name: "1", - args: args{"029c0010517a011588745287047c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d250000b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa38400612709393588893935687000e64635167006868000e7651935188008702effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac7502"}, - want: []string{"9c", "517a011588745287", "709393588893935687", "64635167006868", "76519351880087"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{txidB2T1, 0}, - hexoutpoint{txidB1T1, 3}, - }, - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{txidB1T2, 1}, - }, - }, - }, - { - name: "1", - args: args{"3276A914B434EB0C1A3B7A02E8A29CC616E791EF1E0BF51F88AC003276A9143F8BA3FDA3BA7B69F5818086E12223C6DD25E3C888AC003276A914A08EAE93007F22668AB5E4A9C83C8CD1C325E3E088AC02EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75003276A9148BDF0AA3C567AA5975C2E61321B8BEBBE7293DF688AC0200B2C06055E5E90E9C82BD4181FDE310104391A7FA4F289B1704E5D90CAA3840022EA9144A21DB08FB6882CB152E1FF06780A430740F77048702EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75023276A914CCAAAF374E1B06CB83118453D102587B4273D09588AC003276A9148D802C045445DF49613F6A70DDD2E48526F3701F88AC00"}, - want: []string{"76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac", "76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac", "76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac", "76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac", "a9144a21db08fb6882cb152e1ff06780a430740f770487", "76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac", "76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{txidB1T2, 0}, - }, - []hexoutpoint{ - hexoutpoint{txidB1T1, 1}, - }, - []hexoutpoint{ - hexoutpoint{txidB1T2, 1}, - }, - []hexoutpoint{}, - []hexoutpoint{}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.buf) - if err != nil { - panic(err) - } - got, got2, err := d.unpackBlockAddresses(b) - if (err != nil) != tt.wantErr { - t.Errorf("unpackBlockAddresses() error = %v, wantErr %v", err, tt.wantErr) - return - } - h := make([]string, len(got)) - for i, g := range got { - h[i] = hex.EncodeToString(g) - } - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h, tt.want) - } - h2 := make([][]hexoutpoint, len(got2)) - for i, g := range got2 { - ho := make([]hexoutpoint, len(g)) - for j, o := range g { - ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.index} - } - h2[i] = ho - } - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h2, tt.want2) - } - }) - } -} - func Test_packBigint_unpackBigint(t *testing.T) { bigbig1, _ := big.NewInt(0).SetString("123456789123456789012345", 10) bigbig2, _ := big.NewInt(0).SetString("12345678912345678901234512389012345123456789123456789012345123456789123456789012345", 10) diff --git a/db/sync.go b/db/sync.go index a664e94f..fecfa09c 100644 --- a/db/sync.go +++ b/db/sync.go @@ -346,25 +346,23 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { } // DisconnectBlocks removes all data belonging to blocks in range lower-higher, -// using block data from blockchain, if they are available, -// otherwise doing full scan func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) - // if the chain uses Block to Addresses mapping, always use DisconnectBlockRange - if w.chain.GetChainParser().KeepBlockAddresses() > 0 { - return w.db.DisconnectBlockRange(lower, higher) + // if the chain is UTXO, always use DisconnectBlockRange + if w.chain.GetChainParser().IsUTXOChain() { + return w.db.DisconnectBlockRangeUTXO(lower, higher) } blocks := make([]*bchain.Block, len(hashes)) var err error - // get all blocks first to see if we can avoid full scan + // try to get all blocks first to see if we can avoid full scan for i, hash := range hashes { blocks[i], err = w.chain.GetBlock(hash, 0) if err != nil { // cannot get a block, we must do full range scan - return w.db.DisconnectBlockRange(lower, higher) + return w.db.DisconnectBlockRangeNonUTXO(lower, higher) } } - // then disconnect one after another + // got all blocks to be disconnected, disconnect them one after another for i, block := range blocks { glog.Info("Disconnecting block ", (int(higher) - i), " ", block.Hash) if err = w.db.DisconnectBlock(block); err != nil { From 78d2c085a97805ebefb8b08b974dbeed057eda26 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 15 Aug 2018 16:07:09 +0200 Subject: [PATCH 13/18] Log expected db data version --- db/rocksdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 7ce84b54..aa53d74c 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -116,7 +116,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { - glog.Infof("rocksdb: open %s", path) + glog.Infof("rocksdb: open %s, version %v", path, dbVersion) db, cfh, err := openDB(path) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() From f5b29b3a17e12c98e1f675e6f0f088b441978a05 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 15 Aug 2018 19:22:26 +0200 Subject: [PATCH 14/18] Implement index v3 WIP --- db/rocksdb.go | 187 ++++++++++++++++++++++++++++++++++----------- db/rocksdb_test.go | 40 +++++----- 2 files changed, 167 insertions(+), 60 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index aa53d74c..dfff330a 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -277,15 +277,21 @@ type outpoint struct { index int32 } -type txAddress struct { +type txInput struct { + addrID []byte + vout uint32 + valueSat big.Int +} + +type txOutput struct { addrID []byte spent bool valueSat big.Int } type txAddresses struct { - inputs []txAddress - outputs []txAddress + inputs []txInput + outputs []txOutput } type addrBalance struct { @@ -294,6 +300,17 @@ type addrBalance struct { balanceSat big.Int } +func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText string) { + ad, err := d.chainParser.OutputScriptToAddresses(addrID) + had := hex.EncodeToString(addrID) + if err != nil { + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", had, logText, valueSat.String(), err) + } else { + glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, had, logText, valueSat.String()) + } + valueSat.SetInt64(0) +} + func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { addresses := make(map[string][]outpoint) blockTxids := make([][]byte, len(block.Txs)) @@ -307,7 +324,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } blockTxids[txi] = btxID ta := txAddresses{} - ta.outputs = make([]txAddress, len(tx.Vout)) + ta.outputs = make([]txOutput, len(tx.Vout)) txAddressesMap[string(btxID)] = &ta for i, output := range tx.Vout { tao := &ta.outputs[i] @@ -358,7 +375,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo for txi, tx := range block.Txs { spendingTxid := blockTxids[txi] ta := txAddressesMap[string(spendingTxid)] - ta.inputs = make([]txAddress, len(tx.Vin)) + ta.inputs = make([]txInput, len(tx.Vin)) for i, input := range tx.Vin { tai := &ta.inputs[i] btxID, err := d.chainParser.PackTxid(input.Txid) @@ -392,6 +409,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo continue } tai.addrID = ot.addrID + tai.vout = input.Vout tai.valueSat = ot.valueSat // mark the output as spent in tx ot.spent = true @@ -427,14 +445,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) if ab.balanceSat.Sign() < 0 { - ad, err := d.chainParser.OutputScriptToAddresses(ot.addrID) - had := hex.EncodeToString(ot.addrID) - if err != nil { - glog.Warningf("rocksdb: unparsable address hex '%v' reached negative balance %v, resetting to 0. Parser error %v", had, ab.balanceSat.String(), err) - } else { - glog.Warningf("rocksdb: address %v hex '%v' reached negative balance %v, resetting to 0", ad, had, ab.balanceSat.String()) - } - ab.balanceSat.SetInt64(0) + d.resetValueSatToZero(&ab.balanceSat, ot.addrID, "balance") } ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } @@ -584,25 +595,37 @@ func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { l := packVaruint(uint(len(ta.inputs)), varBuf) buf = append(buf, varBuf[:l]...) for i := range ta.inputs { - buf = appendTxAddress(&ta.inputs[i], buf, varBuf) + buf = appendTxInput(&ta.inputs[i], buf, varBuf) } l = packVaruint(uint(len(ta.outputs)), varBuf) buf = append(buf, varBuf[:l]...) for i := range ta.outputs { - buf = appendTxAddress(&ta.outputs[i], buf, varBuf) + buf = appendTxOutput(&ta.outputs[i], buf, varBuf) } return buf } -func appendTxAddress(txa *txAddress, buf []byte, varBuf []byte) []byte { - la := len(txa.addrID) - if txa.spent { +func appendTxInput(txi *txInput, buf []byte, varBuf []byte) []byte { + la := len(txi.addrID) + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txi.addrID...) + l = packBigint(&txi.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + l = packVaruint(uint(txi.vout), varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func appendTxOutput(txo *txOutput, buf []byte, varBuf []byte) []byte { + la := len(txo.addrID) + if txo.spent { la = ^la } l := packVarint(la, varBuf) buf = append(buf, varBuf[:l]...) - buf = append(buf, txa.addrID...) - l = packBigint(&txa.valueSat, varBuf) + buf = append(buf, txo.addrID...) + l = packBigint(&txo.valueSat, varBuf) buf = append(buf, varBuf[:l]...) return buf } @@ -610,29 +633,41 @@ func appendTxAddress(txa *txAddress, buf []byte, varBuf []byte) []byte { func unpackTxAddresses(buf []byte) (*txAddresses, error) { ta := txAddresses{} inputs, l := unpackVaruint(buf) - ta.inputs = make([]txAddress, inputs) + ta.inputs = make([]txInput, inputs) for i := uint(0); i < inputs; i++ { - l += unpackTxAddress(&ta.inputs[i], buf[l:]) + l += unpackTxInput(&ta.inputs[i], buf[l:]) } outputs, ll := unpackVaruint(buf[l:]) l += ll - ta.outputs = make([]txAddress, outputs) + ta.outputs = make([]txOutput, outputs) for i := uint(0); i < outputs; i++ { - l += unpackTxAddress(&ta.outputs[i], buf[l:]) + l += unpackTxOutput(&ta.outputs[i], buf[l:]) } return &ta, nil } -func unpackTxAddress(ta *txAddress, buf []byte) int { +func unpackTxInput(ti *txInput, buf []byte) int { + al, l := unpackVarint(buf) + ti.addrID = make([]byte, al) + copy(ti.addrID, buf[l:l+al]) + al += l + ti.valueSat, l = unpackBigint(buf[al:]) + al += l + v, l := unpackVaruint(buf[al:]) + ti.vout = uint32(v) + return l + al +} + +func unpackTxOutput(to *txOutput, buf []byte) int { al, l := unpackVarint(buf) if al < 0 { - ta.spent = true + to.spent = true al = ^al } - ta.addrID = make([]byte, al) - copy(ta.addrID, buf[l:l+al]) + to.addrID = make([]byte, al) + copy(to.addrID, buf[l:l+al]) al += l - ta.valueSat, l = unpackBigint(buf[al:]) + to.valueSat, l = unpackBigint(buf[al:]) return l + al } @@ -759,11 +794,7 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { return d.chainParser.UnpackBlockHash(val.Data()) } -func (d *RocksDB) writeHeight( - wb *gorocksdb.WriteBatch, - block *bchain.Block, - op int, -) error { +func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { key := packUint(block.Height) switch op { @@ -776,7 +807,6 @@ func (d *RocksDB) writeHeight( case opDelete: wb.DeleteCF(d.cfh[cfHeight], key) } - return nil } @@ -827,12 +857,46 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, txAddressesToUpdate map[string]*txAddresses, txsToDelete map[string]struct{}, balances map[string]*addrBalance) error { - findBalance := func(addrID []byte) (*addrBalance, error) { +// get all transactions of the given address and match it to input to find spent output +func (d *RocksDB) findSpentTx(ti *txInput) ([]byte, *txAddresses, error) { + start := packAddressKey(ti.addrID, 0) + stop := packAddressKey(ti.addrID, ^uint32(0)) + it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) + defer it.Close() + for it.Seek(start); it.Valid(); it.Next() { + key := it.Key().Data() + val := it.Value().Data() + if bytes.Compare(key, stop) > 0 { + break + } + outpoints, err := d.unpackOutpoints(val) + if err != nil { + return nil, nil, err + } + for _, o := range outpoints { + // process only outputs that match + if o.index >= 0 && uint32(o.index) == ti.vout { + a, err := d.getTxAddresses(o.btxID) + if err != nil { + return nil, nil, err + } + if bytes.Equal(a.outputs[o.index].addrID, ti.addrID) { + return o.btxID, a, nil + } + } + } + } + return nil, nil, nil +} + +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, + txAddressesToUpdate map[string]*txAddresses, balances map[string]*addrBalance) error { + addresses := make(map[string]struct{}) + getAddressBalance := func(addrID []byte) (*addrBalance, error) { var err error s := string(addrID) - b, found := balances[s] - if !found { + b, fb := balances[s] + if !fb { b, err = d.getAddressBalance(addrID) if err != nil { return nil, err @@ -842,15 +906,52 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, return b, nil } for _, t := range txa.inputs { - b, err := findBalance(t.addrID) + s := string(t.addrID) + _, fa := addresses[s] + if !fa { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.addrID) if err != nil { return err } if b != nil { - + // subtract number of txs only once + if !fa { + b.txs-- + } + b.sentSat.Sub(&b.sentSat, &t.valueSat) + if b.sentSat.Sign() < 0 { + d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + } + b.balanceSat.Add(&b.balanceSat, &t.valueSat) } } - + for _, t := range txa.outputs { + s := string(t.addrID) + _, fa := addresses[s] + if !fa { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.addrID) + if err != nil { + return err + } + if b != nil { + // subtract number of txs only once + if !fa { + b.txs-- + } + b.balanceSat.Sub(&b.balanceSat, &t.valueSat) + if b.balanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + } + } + } + for a := range addresses { + key := packAddressKey([]byte(a), height) + wb.DeleteCF(d.cfh[cfAddresses], key) + } return nil } @@ -883,7 +984,7 @@ func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { } s := string(txid) txsToDelete[s] = struct{}{} - if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, txsToDelete, balances); err != nil { + if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, balances); err != nil { return err } } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 4a96e769..fe841e82 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -432,8 +432,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T1, "02" + - addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + - addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + "00" + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "01" + "02" + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), @@ -442,8 +442,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T2, "02" + - addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + - addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + "00" + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "01" + "02" + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), @@ -452,7 +452,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T3, "01" + - addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "02" + "01" + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, @@ -548,7 +548,7 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality // 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using blockaddresses column +// 6) Disconnect the block 2 using BlockTxids column // 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { @@ -762,19 +762,21 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }{ { name: "1", - hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b7002c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b700e0392c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a19987283bb55012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", data: &txAddresses{ - inputs: []txAddress{ + inputs: []txInput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), valueSat: *big.NewInt(0), + vout: 12345, }, { addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), valueSat: *big.NewInt(1234123421342341234), + vout: 56789, }, }, - outputs: []txAddress{ + outputs: []txOutput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), valueSat: *big.NewInt(1), @@ -785,23 +787,26 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "2", - hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c02ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c02ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c0012ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c0022ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec003052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", data: &txAddresses{ - inputs: []txAddress{ + inputs: []txInput{ { addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), valueSat: *big.NewInt(9011000000), + vout: 1, }, { addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), valueSat: *big.NewInt(8011000000), + vout: 2, }, { addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), valueSat: *big.NewInt(7011000000), + vout: 3, }, }, - outputs: []txAddress{ + outputs: []txOutput{ { addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), valueSat: *big.NewInt(5011000000), @@ -830,15 +835,16 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "empty address", - hex: "01000204d2020002162e010162", + hex: "01000204d201020002162e010162", data: &txAddresses{ - inputs: []txAddress{ + inputs: []txInput{ { addrID: []byte{}, valueSat: *big.NewInt(1234), + vout: 1, }, }, - outputs: []txAddress{ + outputs: []txOutput{ { addrID: []byte{}, valueSat: *big.NewInt(5678), @@ -855,8 +861,8 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { name: "empty", hex: "0000", data: &txAddresses{ - inputs: []txAddress{}, - outputs: []txAddress{}, + inputs: []txInput{}, + outputs: []txOutput{}, }, }, } From 745d7af92d65718370233884a55f179d2173509c Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 16 Aug 2018 15:31:11 +0200 Subject: [PATCH 15/18] Implement index v3 WIP --- db/rocksdb.go | 280 ++++++++++++++++++++++++++------------------- db/rocksdb_test.go | 79 ++++++++----- 2 files changed, 215 insertions(+), 144 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index dfff330a..08d1eb78 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -50,11 +50,11 @@ const ( cfAddresses cfTxAddresses cfAddressBalance - cfBlockTxids + cfBlockTxs cfTransactions ) -var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxids", "transactions"} +var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -279,7 +279,6 @@ type outpoint struct { type txInput struct { addrID []byte - vout uint32 valueSat big.Int } @@ -300,6 +299,11 @@ type addrBalance struct { balanceSat big.Int } +type blockTxs struct { + btxID []byte + inputs []outpoint +} + func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText string) { ad, err := d.chainParser.OutputScriptToAddresses(addrID) had := hex.EncodeToString(addrID) @@ -313,16 +317,17 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { addresses := make(map[string][]outpoint) - blockTxids := make([][]byte, len(block.Txs)) + blockTxIDs := make([][]byte, len(block.Txs)) txAddressesMap := make(map[string]*txAddresses) balances := make(map[string]*addrBalance) // first process all outputs so that inputs can point to txs in this block - for txi, tx := range block.Txs { + for txi := range block.Txs { + tx := &block.Txs[txi] btxID, err := d.chainParser.PackTxid(tx.Txid) if err != nil { return err } - blockTxids[txi] = btxID + blockTxIDs[txi] = btxID ta := txAddresses{} ta.outputs = make([]txOutput, len(tx.Vout)) txAddressesMap[string(btxID)] = &ta @@ -372,8 +377,9 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } } // process inputs - for txi, tx := range block.Txs { - spendingTxid := blockTxids[txi] + for txi := range block.Txs { + tx := &block.Txs[txi] + spendingTxid := blockTxIDs[txi] ta := txAddressesMap[string(spendingTxid)] ta.inputs = make([]txInput, len(tx.Vin)) for i, input := range tx.Vin { @@ -406,10 +412,8 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ot := &ita.outputs[int(input.Vout)] if ot.spent { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) - continue } tai.addrID = ot.addrID - tai.vout = input.Vout tai.valueSat = ot.valueSat // mark the output as spent in tx ot.spent = true @@ -459,7 +463,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo if err := d.storeBalances(wb, balances); err != nil { return err } - return d.storeAndCleanupBlockTxids(wb, block, blockTxids) + return d.storeAndCleanupBlockTxs(wb, block) } func processedInTx(o []outpoint, btxID []byte) bool { @@ -510,22 +514,38 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa return nil } -func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bchain.Block, txids [][]byte) error { - pl := d.chainParser.PackedTxidLen() - buf := make([]byte, pl*len(txids)) - i := 0 - for _, txid := range txids { - copy(buf[i:], txid) - i += pl +func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { + buf := make([]byte, 0, d.chainParser.PackedTxidLen()*len(block.Txs)) + varBuf := make([]byte, vlq.MaxLen64) + for i := range block.Txs { + tx := &block.Txs[i] + o := make([]outpoint, len(tx.Vin)) + for v := range tx.Vin { + vin := &tx.Vin[v] + btxID, err := d.chainParser.PackTxid(vin.Txid) + if err != nil { + return err + } + o[v].btxID = btxID + o[v].index = int32(vin.Vout) + } + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return err + } + buf = append(buf, btxID...) + l := packVaruint(uint(len(o)), varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, d.packOutpoints(o)...) } key := packUint(block.Height) - wb.PutCF(d.cfh[cfBlockTxids], key, buf) + wb.PutCF(d.cfh[cfBlockTxs], key, buf) keep := d.chainParser.KeepBlockAddresses() // cleanup old block address if block.Height > uint32(keep) { for rh := block.Height - uint32(keep); rh < block.Height; rh-- { key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], key) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) if err != nil { return err } @@ -533,26 +553,41 @@ func (d *RocksDB) storeAndCleanupBlockTxids(wb *gorocksdb.WriteBatch, block *bch break } val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockTxids], key) + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) } } return nil } -func (d *RocksDB) getBlockTxids(height uint32) ([][]byte, error) { +func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { pl := d.chainParser.PackedTxidLen() - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxids], packUint(height)) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) if err != nil { return nil, err } defer val.Free() buf := val.Data() - txids := make([][]byte, len(buf)/pl) - for i := 0; i < len(txids); i++ { + bt := make([]blockTxs, 0) + for i := 0; i < len(buf); { + if len(buf)-i < pl { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } txid := make([]byte, pl) - copy(txid, buf[pl*i:]) + copy(txid, buf[i:]) + i += pl + o, ol, err := d.unpackNOutpoints(buf[i:]) + if err != nil { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + bt = append(bt, blockTxs{ + btxID: txid, + inputs: o, + }) + i += ol } - return txids, nil + return bt, nil } func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { @@ -607,13 +642,11 @@ func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { func appendTxInput(txi *txInput, buf []byte, varBuf []byte) []byte { la := len(txi.addrID) - l := packVarint(la, varBuf) + l := packVaruint(uint(la), varBuf) buf = append(buf, varBuf[:l]...) buf = append(buf, txi.addrID...) l = packBigint(&txi.valueSat, varBuf) buf = append(buf, varBuf[:l]...) - l = packVaruint(uint(txi.vout), varBuf) - buf = append(buf, varBuf[:l]...) return buf } @@ -647,15 +680,12 @@ func unpackTxAddresses(buf []byte) (*txAddresses, error) { } func unpackTxInput(ti *txInput, buf []byte) int { - al, l := unpackVarint(buf) + al, l := unpackVaruint(buf) ti.addrID = make([]byte, al) - copy(ti.addrID, buf[l:l+al]) - al += l + copy(ti.addrID, buf[l:l+int(al)]) + al += uint(l) ti.valueSat, l = unpackBigint(buf[al:]) - al += l - v, l := unpackVaruint(buf[al:]) - ti.vout = uint32(v) - return l + al + return l + int(al) } func unpackTxOutput(to *txOutput, buf []byte) int { @@ -698,6 +728,26 @@ func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { return outpoints, nil } +func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + n, p := unpackVaruint(buf) + outpoints := make([]outpoint, n) + for i := uint(0); i < n; i++ { + if p+txidUnpackedLen >= len(buf) { + return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") + } + btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) + p += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[p:]) + p += voutLen + outpoints[i] = outpoint{ + btxID: btxID, + index: vout, + } + } + return outpoints, p, nil +} + func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { if len(addrID) > 0 { if len(addrID) > maxAddrIDLen { @@ -857,39 +907,7 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -// get all transactions of the given address and match it to input to find spent output -func (d *RocksDB) findSpentTx(ti *txInput) ([]byte, *txAddresses, error) { - start := packAddressKey(ti.addrID, 0) - stop := packAddressKey(ti.addrID, ^uint32(0)) - it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) - defer it.Close() - for it.Seek(start); it.Valid(); it.Next() { - key := it.Key().Data() - val := it.Value().Data() - if bytes.Compare(key, stop) > 0 { - break - } - outpoints, err := d.unpackOutpoints(val) - if err != nil { - return nil, nil, err - } - for _, o := range outpoints { - // process only outputs that match - if o.index >= 0 && uint32(o.index) == ti.vout { - a, err := d.getTxAddresses(o.btxID) - if err != nil { - return nil, nil, err - } - if bytes.Equal(a.outputs[o.index].addrID, ti.addrID) { - return o.btxID, a, nil - } - } - } - } - return nil, nil, nil -} - -func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, txa *txAddresses, +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *txAddresses, txAddressesToUpdate map[string]*txAddresses, balances map[string]*addrBalance) error { addresses := make(map[string]struct{}) getAddressBalance := func(addrID []byte) (*addrBalance, error) { @@ -905,46 +923,68 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } return b, nil } - for _, t := range txa.inputs { - s := string(t.addrID) - _, fa := addresses[s] - if !fa { - addresses[s] = struct{}{} - } - b, err := getAddressBalance(t.addrID) - if err != nil { - return err - } - if b != nil { - // subtract number of txs only once - if !fa { - b.txs-- + for i, t := range txa.inputs { + if len(t.addrID) > 0 { + s := string(t.addrID) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} } - b.sentSat.Sub(&b.sentSat, &t.valueSat) - if b.sentSat.Sign() < 0 { - d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + b, err := getAddressBalance(t.addrID) + if err != nil { + return err } - b.balanceSat.Add(&b.balanceSat, &t.valueSat) + if b != nil { + // subtract number of txs only once + if !exist { + b.txs-- + } + b.sentSat.Sub(&b.sentSat, &t.valueSat) + if b.sentSat.Sign() < 0 { + d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + } + b.balanceSat.Add(&b.balanceSat, &t.valueSat) + } else { + ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) + had := hex.EncodeToString(t.addrID) + glog.Warningf("Balance for address %s (%s) not found", ad, had) + } + s = string(inputs[i].btxID) + sa, exist := txAddressesToUpdate[s] + if !exist { + sa, err = d.getTxAddresses(inputs[i].btxID) + if err != nil { + return err + } + txAddressesToUpdate[s] = sa + } + sa.outputs[inputs[i].index].spent = false } } for _, t := range txa.outputs { - s := string(t.addrID) - _, fa := addresses[s] - if !fa { - addresses[s] = struct{}{} - } - b, err := getAddressBalance(t.addrID) - if err != nil { - return err - } - if b != nil { - // subtract number of txs only once - if !fa { - b.txs-- + if len(t.addrID) > 0 { + s := string(t.addrID) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} } - b.balanceSat.Sub(&b.balanceSat, &t.valueSat) - if b.balanceSat.Sign() < 0 { - d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + b, err := getAddressBalance(t.addrID) + if err != nil { + return err + } + if b != nil { + // subtract number of txs only once + if !exist { + b.txs-- + } + b.balanceSat.Sub(&b.balanceSat, &t.valueSat) + if b.balanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + } + } else { + ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) + had := hex.EncodeToString(t.addrID) + glog.Warningf("Balance for address %s (%s) not found", ad, had) } } } @@ -956,19 +996,19 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, } // DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher -// if they are in the range kept in the cfBlockTxs column +// if they are in the range kept in the cfBlockTxids column func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - blocksTxids := make([][][]byte, higher-lower+1) + blocks := make([][]blockTxs, higher-lower+1) for height := lower; height <= higher; height++ { - blockTxids, err := d.getBlockTxids(height) + blockTxs, err := d.getBlockTxs(height) if err != nil { return err } - if len(blockTxids) == 0 { + if len(blockTxs) == 0 { return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) } - blocksTxids[height-lower] = blockTxids + blocks[height-lower] = blockTxs } wb := gorocksdb.NewWriteBatch() defer wb.Destroy() @@ -976,20 +1016,30 @@ func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { txsToDelete := make(map[string]struct{}) balances := make(map[string]*addrBalance) for height := higher; height >= lower; height-- { - blockTxids := blocksTxids[height-lower] - for _, txid := range blockTxids { + blockTxs := blocks[height-lower] + glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") + // go backwards to avoid interim negative balance + // when connecting block, amount is first in tx on the output side, then in another tx on the input side + // when disconnecting, it must be done backwards + for i := len(blockTxs) - 1; i >= 0; i-- { + txid := blockTxs[i].btxID + s := string(txid) + txsToDelete[s] = struct{}{} txa, err := d.getTxAddresses(txid) if err != nil { return err } - s := string(txid) - txsToDelete[s] = struct{}{} - if err := d.disconnectTxAddresses(wb, height, s, txa, txAddressesToUpdate, balances); err != nil { + if txa == nil { + ut, _ := d.chainParser.UnpackTxid(txid) + glog.Warning("TxAddress for txid ", ut, " not found") + continue + } + if err := d.disconnectTxAddresses(wb, height, s, blockTxs[i].inputs, txa, txAddressesToUpdate, balances); err != nil { return err } } key := packUint(height) - wb.DeleteCF(d.cfh[cfBlockTxids], key) + wb.DeleteCF(d.cfh[cfBlockTxs], key) wb.DeleteCF(d.cfh[cfHeight], key) } d.storeTxAddresses(wb, txAddressesToUpdate) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index fe841e82..7157931e 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -24,10 +24,9 @@ import ( // takes only 1 byte if abs(n)<127 func bitcoinTestnetParser() *btc.BitcoinParser { - return &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, - Params: btc.GetChainParams("test"), - } + return btc.NewBitcoinParser( + btc.GetChainParams("test"), + &btc.Configuration{BlockAddressesToKeep: 1}) } func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { @@ -62,6 +61,11 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { return hex.EncodeToString(b) } +func inputAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return strconv.FormatInt(int64(len(h)/2), 16) + h +} + func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) return strconv.FormatInt(int64(len(h)), 16) + h @@ -324,7 +328,7 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { } } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, }); err != nil { @@ -376,9 +380,21 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockTxids, []keyPair{ - keyPair{"000370d5", txidB1T1 + txidB1T2, nil}, - }); err != nil { + + var blockTxsKp []keyPair + if afterDisconnect { + blockTxsKp = []keyPair{} + } else { + blockTxsKp = []keyPair{ + keyPair{ + "000370d5", + txidB1T1 + "00" + txidB1T2 + "00", + nil, + }, + } + } + + if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil { { t.Fatal(err) } @@ -432,8 +448,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T1, "02" + - addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + "00" + - addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "01" + + inputAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + inputAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + "02" + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), @@ -442,8 +458,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T2, "02" + - addressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + "00" + - addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "01" + + inputAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + inputAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + "02" + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), @@ -452,7 +468,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{ txidB2T3, "01" + - addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "02" + + inputAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + "01" + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, @@ -477,8 +493,14 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockTxids, []keyPair{ - keyPair{"000370d6", txidB2T1 + txidB2T2 + txidB2T3, nil}, + if err := checkColumn(d, cfBlockTxs, []keyPair{ + keyPair{ + "000370d6", + txidB2T1 + "02" + txidB1T2 + "00" + txidB1T1 + "02" + + txidB2T2 + "02" + txidB2T1 + "00" + txidB1T2 + "02" + + txidB2T3 + "01" + txidB1T2 + "04", + nil, + }, }); err != nil { { t.Fatal(err) @@ -548,8 +570,8 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality // 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using BlockTxids column -// 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error +// 6) Disconnect the block 2 using BlockTxs column +// 7) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ @@ -562,7 +584,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d) + verifyAfterUTXOBlock1(t, d, false) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block block2 := getTestUTXOBlock2(t, d) @@ -645,14 +667,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { if err != nil { t.Fatal(err) } - - verifyAfterUTXOBlock1(t, d) + verifyAfterUTXOBlock1(t, d, true) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { t.Fatal(err) } } + // connect block again and verify the state of db + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + } func Test_packBigint_unpackBigint(t *testing.T) { @@ -762,18 +789,16 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }{ { name: "1", - hex: "022c001443aac20a116e09ea4f7914be1c55e4c17aa600b700e0392c001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a19987283bb55012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + hex: "0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", data: &txAddresses{ inputs: []txInput{ { addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), valueSat: *big.NewInt(0), - vout: 12345, }, { addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), valueSat: *big.NewInt(1234123421342341234), - vout: 56789, }, }, outputs: []txOutput{ @@ -787,23 +812,20 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "2", - hex: "032ea9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c0012ea91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c0022ea914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec003052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + hex: "0317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", data: &txAddresses{ inputs: []txInput{ { addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), valueSat: *big.NewInt(9011000000), - vout: 1, }, { addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), valueSat: *big.NewInt(8011000000), - vout: 2, }, { addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), valueSat: *big.NewInt(7011000000), - vout: 3, }, }, outputs: []txOutput{ @@ -835,13 +857,12 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { }, { name: "empty address", - hex: "01000204d201020002162e010162", + hex: "01000204d2020002162e010162", data: &txAddresses{ inputs: []txInput{ { addrID: []byte{}, valueSat: *big.NewInt(1234), - vout: 1, }, }, outputs: []txOutput{ From 878d25ea426c33ba59cac3a3c782ed8068966a13 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 17 Aug 2018 23:48:36 +0200 Subject: [PATCH 16/18] Fix handling of coinbase txs in db, add test for it --- db/rocksdb.go | 11 ++++++++-- db/rocksdb_test.go | 51 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 08d1eb78..74ebb3e1 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -515,8 +515,10 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBa } func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { - buf := make([]byte, 0, d.chainParser.PackedTxidLen()*len(block.Txs)) + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, 0, pl*len(block.Txs)) varBuf := make([]byte, vlq.MaxLen64) + zeroTx := make([]byte, pl) for i := range block.Txs { tx := &block.Txs[i] o := make([]outpoint, len(tx.Vin)) @@ -524,7 +526,12 @@ func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchai vin := &tx.Vin[v] btxID, err := d.chainParser.PackTxid(vin.Txid) if err != nil { - return err + // do not process inputs without input txid + if err == bchain.ErrTxidMissing { + btxID = zeroTx + } else { + return err + } } o[v].btxID = btxID o[v].index = int32(vin.Vout) diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 7157931e..3ffe7e11 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -12,7 +12,6 @@ import ( "os" "reflect" "sort" - "strconv" "strings" "testing" @@ -54,6 +53,9 @@ func closeAndDestroyRocksDB(t *testing.T, d *RocksDB) { } func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { + if addr == "" { + return "" + } b, err := d.chainParser.AddressToOutputScript(addr) if err != nil { t.Fatal(err) @@ -63,17 +65,17 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { func inputAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)/2), 16) + h + return hex.EncodeToString([]byte{byte(len(h) / 2)}) + h } func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)), 16) + h + return hex.EncodeToString([]byte{byte(len(h))}) + h } func spentAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)+1), 16) + h + return hex.EncodeToString([]byte{byte(len(h) + 1)}) + h } func bigintToHex(i *big.Int) string { @@ -143,6 +145,7 @@ const ( txidB2T1 = "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" txidB2T2 = "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" txidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + txidB2T4 = "fdd824a780cbb718eeb766eb05d83fdefc793a27082cd5e67f856d69798cf7db" addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac @@ -153,6 +156,7 @@ const ( addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac + addrA = "mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj" // 76a914d03c0d863d189b23b061a95ad32940b65837609f88ac ) var ( @@ -167,6 +171,7 @@ var ( satB2T2A8 = big.NewInt(118641975500) satB2T2A9 = big.NewInt(198641975500) satB2T3A5 = big.NewInt(9000) + satB2T4AA = big.NewInt(1360030331) ) func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { @@ -324,6 +329,31 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { Blocktime: 22549400002, Time: 22549400002, }, + // mining transaction + bchain.Tx{ + Txid: txidB2T4, + Vin: []bchain.Vin{ + bchain.Vin{ + Coinbase: "03bf1e1504aede765b726567696f6e312f50726f6a65637420425443506f6f6c2f01000001bf7e000000000000", + }, + }, + Vout: []bchain.Vout{ + bchain.Vout{ + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex(addrA, t, d), + }, + ValueSat: *satB2T4AA, + }, + bchain.Vout{ + N: 1, + ScriptPubKey: bchain.ScriptPubKey{}, + ValueSat: *satZero, + }, + }, + Blocktime: 22549400003, + Time: 22549400003, + }, }, } } @@ -423,6 +453,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{addressToPubKeyHex(addr3, t, d) + "000370d6", txidB2T1 + "01", nil}, keyPair{addressToPubKeyHex(addr2, t, d) + "000370d6", txidB2T1 + "03", nil}, keyPair{addressToPubKeyHex(addr5, t, d) + "000370d6", txidB2T3 + "00" + txidB2T3 + "01", nil}, + keyPair{addressToPubKeyHex(addrA, t, d) + "000370d6", txidB2T4 + "00", nil}, keyPair{addressToPubKeyHex(addr4, t, d) + "000370d6", txidB2T2 + "03", nil}, }); err != nil { { @@ -473,6 +504,14 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), nil, }, + keyPair{ + txidB2T4, + "01" + inputAddressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero) + + "02" + + addressToPubKeyHexWithLength(addrA, t, d) + bigintToHex(satB2T4AA) + + addressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero), + nil, + }, }); err != nil { { t.Fatal(err) @@ -488,6 +527,7 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { keyPair{addressToPubKeyHex(addr7, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T1A7), nil}, keyPair{addressToPubKeyHex(addr8, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A8), nil}, keyPair{addressToPubKeyHex(addr9, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A9), nil}, + keyPair{addressToPubKeyHex(addrA, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T4AA), nil}, }); err != nil { { t.Fatal(err) @@ -498,7 +538,8 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { "000370d6", txidB2T1 + "02" + txidB1T2 + "00" + txidB1T1 + "02" + txidB2T2 + "02" + txidB2T1 + "00" + txidB1T2 + "02" + - txidB2T3 + "01" + txidB1T2 + "04", + txidB2T3 + "01" + txidB1T2 + "04" + + txidB2T4 + "01" + "0000000000000000000000000000000000000000000000000000000000000000" + "00", nil, }, }); err != nil { From d45d028ef229cfe11875afd18a8d823b71aedc25 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 19 Aug 2018 00:23:26 +0200 Subject: [PATCH 17/18] Implement bulk connect blocks --- blockbook.go | 6 +- common/internalstate.go | 2 + db/rocksdb.go | 225 ++++++++++++++++++++++++++++++++++++---- db/rocksdb_test.go | 39 +++++++ db/sync.go | 15 ++- 5 files changed, 266 insertions(+), 21 deletions(-) diff --git a/blockbook.go b/blockbook.go index cea2c5b6..7e3da52b 100644 --- a/blockbook.go +++ b/blockbook.go @@ -175,7 +175,11 @@ func main() { } index.SetInternalState(internalState) if internalState.DbState != common.DbStateClosed { - glog.Warning("internalState: database in not closed state ", internalState.DbState, ", possibly previous ungraceful shutdown") + if internalState.DbState == common.DbStateInconsistent { + glog.Error("internalState: database is in inconsistent state and cannot be used") + return + } + glog.Warning("internalState: database was left in open state, possibly previous ungraceful shutdown") } if *computeColumnStats { diff --git a/common/internalstate.go b/common/internalstate.go index 8e7b2b61..1053649c 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -11,6 +11,8 @@ const ( DbStateClosed = uint32(iota) // DbStateOpen means db is open or application died without closing the db DbStateOpen + // DbStateInconsistent means db is in inconsistent state and cannot be used + DbStateInconsistent ) // InternalStateColumn contains the data of a db column diff --git a/db/rocksdb.go b/db/rocksdb.go index 74ebb3e1..267a0ff6 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -258,7 +258,18 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { // unspentTxs; therefore it is not possible to DisconnectBlocks this way return errors.New("DisconnectBlock is not supported for UTXO chains") } - if err := d.writeAddressesUTXO(wb, block); err != nil { + txAddressesMap := make(map[string]*txAddresses) + balances := make(map[string]*addrBalance) + if err := d.writeAddressesUTXO(wb, block, txAddressesMap, balances); err != nil { + return err + } + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { return err } } else { @@ -270,6 +281,177 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return d.db.Write(d.wo, wb) } +// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type BulkConnect struct { + d *RocksDB + isUTXO bool + txAddressesMap map[string]*txAddresses + balances map[string]*addrBalance + height uint32 +} + +const ( + maxBulkTxAddresses = 2000000 + partialStoreAddresses = maxBulkTxAddresses / 10 + maxBulkBalances = 2500000 + partialStoreBalances = maxBulkBalances / 10 +) + +func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { + bc := &BulkConnect{ + d: d, + isUTXO: d.chainParser.IsUTXOChain(), + txAddressesMap: make(map[string]*txAddresses), + balances: make(map[string]*addrBalance), + } + if err := d.SetInconsistentState(true); err != nil { + return nil, err + } + glog.Info("rocksdb: bulk connect init, db set to inconsistent state") + return bc, nil +} + +func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { + defer close(c) + start := time.Now() + var txm map[string]*txAddresses + if all { + txm = b.txAddressesMap + b.txAddressesMap = make(map[string]*txAddresses) + } else { + txm = make(map[string]*txAddresses) + for k, a := range b.txAddressesMap { + // store all completely spent transactions, they will not be modified again + r := true + for _, o := range a.outputs { + if o.spent == false { + r = false + break + } + } + if r { + txm[k] = a + delete(b.txAddressesMap, k) + } + } + // store some other random transactions if necessary + if len(txm) < partialStoreAddresses { + for k, a := range b.txAddressesMap { + txm[k] = a + delete(b.txAddressesMap, k) + if len(txm) >= partialStoreAddresses { + break + } + } + } + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.storeTxAddresses(wb, txm); err != nil { + c <- err + } else { + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + } + } + glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) +} + +func (b *BulkConnect) storeBalances(c chan error, all bool) { + defer close(c) + start := time.Now() + var bal map[string]*addrBalance + if all { + bal = b.balances + b.balances = make(map[string]*addrBalance) + } else { + bal = make(map[string]*addrBalance) + // store some random balances + for k, a := range b.balances { + bal[k] = a + delete(b.balances, k) + if len(bal) >= partialStoreBalances { + break + } + } + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.storeBalances(wb, bal); err != nil { + c <- err + } else { + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + } + } + glog.Info("rocksdb: height ", b.height, ", stored ", len(bal), " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) +} + +func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { + b.height = block.Height + if !b.isUTXO { + return b.d.ConnectBlock(block) + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.writeAddressesUTXO(wb, block, b.txAddressesMap, b.balances); err != nil { + return err + } + var storeAddressesChan, storeBalancesChan chan error + if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { + storeAddressesChan = make(chan error) + go b.storeTxAddresses(storeAddressesChan, false) + } + if len(b.balances)+partialStoreBalances > maxBulkBalances { + storeBalancesChan = make(chan error) + go b.storeBalances(storeBalancesChan, false) + } + } + if storeBlockTxs { + if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { + return err + } + } + if err := b.d.writeHeight(wb, block, opInsert); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + if storeAddressesChan != nil { + if err := <-storeAddressesChan; err != nil { + return err + } + } + if storeBalancesChan != nil { + if err := <-storeBalancesChan; err != nil { + return err + } + } + return nil +} + +func (b *BulkConnect) Close() error { + glog.Info("rocksdb: bulk connect closing") + storeAddressesChan := make(chan error) + go b.storeTxAddresses(storeAddressesChan, true) + storeBalancesChan := make(chan error) + go b.storeBalances(storeBalancesChan, true) + if err := <-storeAddressesChan; err != nil { + return err + } + if err := <-storeBalancesChan; err != nil { + return err + } + if err := b.d.SetInconsistentState(false); err != nil { + return err + } + glog.Info("rocksdb: bulk connect closed, db set to open state") + b.d = nil + return nil +} + // Addresses index type outpoint struct { @@ -315,11 +497,9 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText valueSat.SetInt64(0) } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block) error { +func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { addresses := make(map[string][]outpoint) blockTxIDs := make([][]byte, len(block.Txs)) - txAddressesMap := make(map[string]*txAddresses) - balances := make(map[string]*addrBalance) // first process all outputs so that inputs can point to txs in this block for txi := range block.Txs { tx := &block.Txs[txi] @@ -454,16 +634,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } } - if err := d.storeAddresses(wb, block, addresses); err != nil { - return err - } - if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { - return err - } - if err := d.storeBalances(wb, balances); err != nil { - return err - } - return d.storeAndCleanupBlockTxs(wb, block) + return d.storeAddresses(wb, block, addresses) } func processedInTx(o []outpoint, btxID []byte) bool { @@ -1228,6 +1399,18 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro return is, nil } +func (d *RocksDB) SetInconsistentState(inconsistent bool) error { + if d.is == nil { + return errors.New("Internal state not created") + } + if inconsistent { + d.is.DbState = common.DbStateInconsistent + } else { + d.is.DbState = common.DbStateOpen + } + return d.storeState(d.is) +} + // SetInternalState sets the InternalState to be used by db to collect internal state func (d *RocksDB) SetInternalState(is *common.InternalState) { d.is = is @@ -1235,11 +1418,17 @@ func (d *RocksDB) SetInternalState(is *common.InternalState) { // StoreInternalState stores the internal state to db func (d *RocksDB) StoreInternalState(is *common.InternalState) error { - for c := 0; c < len(cfNames); c++ { - rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) - d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) - d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + if d.metrics != nil { + for c := 0; c < len(cfNames); c++ { + rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) + d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) + d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + } } + return d.storeState(is) +} + +func (d *RocksDB) storeState(is *common.InternalState) error { buf, err := is.Pack() if err != nil { return err diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 3ffe7e11..adf9e6c9 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -5,6 +5,7 @@ package db import ( "blockbook/bchain" "blockbook/bchain/coins/btc" + "blockbook/common" "encoding/hex" "fmt" "io/ioutil" @@ -720,7 +721,45 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } verifyAfterUTXOBlock2(t, d) +} +func Test_BulkConnect_UTXO(t *testing.T) { + d := setupRocksDB(t, &testBitcoinParser{ + BitcoinParser: bitcoinTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + bc, err := d.InitBulkConnect() + if err != nil { + t.Fatal(err) + } + + if d.is.DbState != common.DbStateInconsistent { + t.Fatal("DB not in DbStateInconsistent") + } + + if err := bc.ConnectBlock(getTestUTXOBlock1(t, d), false); err != nil { + t.Fatal(err) + } + if err := checkColumn(d, cfBlockTxs, []keyPair{}); err != nil { + { + t.Fatal(err) + } + } + + if err := bc.ConnectBlock(getTestUTXOBlock2(t, d), true); err != nil { + t.Fatal(err) + } + + if err := bc.Close(); err != nil { + t.Fatal(err) + } + + if d.is.DbState != common.DbStateOpen { + t.Fatal("DB not in DbStateOpen") + } + + verifyAfterUTXOBlock2(t, d) } func Test_packBigint_unpackBigint(t *testing.T) { diff --git a/db/sync.go b/db/sync.go index fecfa09c..32bc143d 100644 --- a/db/sync.go +++ b/db/sync.go @@ -213,17 +213,26 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { writeBlockDone := make(chan struct{}) writeBlockWorker := func() { defer close(writeBlockDone) + bc, err := w.db.InitBulkConnect() + if err != nil { + glog.Error("sync: InitBulkConnect error ", err) + } lastBlock := lower - 1 + keep := uint32(w.chain.GetChainParser().KeepBlockAddresses()) for b := range bch { if lastBlock+1 != b.Height { glog.Error("writeBlockWorker skipped block, last connected block", lastBlock, ", new block ", b.Height) } - err := w.db.ConnectBlock(b) + err := bc.ConnectBlock(b, b.Height+keep > higher) if err != nil { glog.Error("writeBlockWorker ", b.Height, " ", b.Hash, " error ", err) } lastBlock = b.Height } + err = bc.Close() + if err != nil { + glog.Error("sync: bulkconnect.Close error ", err) + } glog.Info("WriteBlock exiting...") } getBlockWorker := func(i int) { @@ -276,6 +285,7 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { } go writeBlockWorker() var hash string + start := time.Now() ConnectLoop: for h := lower; h <= higher; { select { @@ -292,7 +302,8 @@ ConnectLoop: } hch <- hashHeight{hash, h} if h > 0 && h%1000 == 0 { - glog.Info("connecting block ", h, " ", hash) + glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start)) + start = time.Now() } h++ } From 5621ed49f3b642adf996c2d8dc1d1011bd526d89 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Sun, 19 Aug 2018 17:43:00 +0200 Subject: [PATCH 18/18] Optimize bulk connect of blocks --- db/rocksdb.go | 79 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/db/rocksdb.go b/db/rocksdb.go index 267a0ff6..1141df55 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -258,9 +258,13 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { // unspentTxs; therefore it is not possible to DisconnectBlocks this way return errors.New("DisconnectBlock is not supported for UTXO chains") } + addresses := make(map[string][]outpoint) txAddressesMap := make(map[string]*txAddresses) balances := make(map[string]*addrBalance) - if err := d.writeAddressesUTXO(wb, block, txAddressesMap, balances); err != nil { + if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { + return err + } + if err := d.storeAddresses(wb, block.Height, addresses); err != nil { return err } if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { @@ -282,15 +286,23 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { } // BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type bulkAddresses struct { + height uint32 + addresses map[string][]outpoint +} + type BulkConnect struct { - d *RocksDB - isUTXO bool - txAddressesMap map[string]*txAddresses - balances map[string]*addrBalance - height uint32 + d *RocksDB + isUTXO bool + bulkAddresses []bulkAddresses + bulkAddressesCount int + txAddressesMap map[string]*txAddresses + balances map[string]*addrBalance + height uint32 } const ( + maxBulkAddresses = 400000 maxBulkTxAddresses = 2000000 partialStoreAddresses = maxBulkTxAddresses / 10 maxBulkBalances = 2500000 @@ -315,6 +327,7 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { defer close(c) start := time.Now() var txm map[string]*txAddresses + var sp int if all { txm = b.txAddressesMap b.txAddressesMap = make(map[string]*txAddresses) @@ -334,6 +347,7 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { delete(b.txAddressesMap, k) } } + sp = len(txm) // store some other random transactions if necessary if len(txm) < partialStoreAddresses { for k, a := range b.txAddressesMap { @@ -354,7 +368,7 @@ func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { c <- err } } - glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) } func (b *BulkConnect) storeBalances(c chan error, all bool) { @@ -387,6 +401,17 @@ func (b *BulkConnect) storeBalances(c chan error, all bool) { glog.Info("rocksdb: height ", b.height, ", stored ", len(bal), " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) } +func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { + for _, ba := range b.bulkAddresses { + if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { + return err + } + } + b.bulkAddressesCount = 0 + b.bulkAddresses = b.bulkAddresses[:0] + return nil +} + func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { b.height = block.Height if !b.isUTXO { @@ -394,11 +419,15 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro } wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - if err := b.d.writeAddressesUTXO(wb, block, b.txAddressesMap, b.balances); err != nil { + addresses := make(map[string][]outpoint) + if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { return err } + start := time.Now() + var sa bool var storeAddressesChan, storeBalancesChan chan error if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + sa = true if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { storeAddressesChan = make(chan error) go b.storeTxAddresses(storeAddressesChan, false) @@ -408,6 +437,17 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro go b.storeBalances(storeBalancesChan, false) } } + b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ + height: block.Height, + addresses: addresses, + }) + b.bulkAddressesCount += len(addresses) + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + } if storeBlockTxs { if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { return err @@ -419,6 +459,9 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro if err := b.d.db.Write(b.d.wo, wb); err != nil { return err } + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + } if storeAddressesChan != nil { if err := <-storeAddressesChan; err != nil { return err @@ -434,10 +477,21 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro func (b *BulkConnect) Close() error { glog.Info("rocksdb: bulk connect closing") + start := time.Now() storeAddressesChan := make(chan error) go b.storeTxAddresses(storeAddressesChan, true) storeBalancesChan := make(chan error) go b.storeBalances(storeBalancesChan, true) + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) if err := <-storeAddressesChan; err != nil { return err } @@ -497,8 +551,7 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText valueSat.SetInt64(0) } -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { - addresses := make(map[string][]outpoint) +func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { blockTxIDs := make([][]byte, len(block.Txs)) // first process all outputs so that inputs can point to txs in this block for txi := range block.Txs { @@ -634,7 +687,7 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo ab.sentSat.Add(&ab.sentSat, &ot.valueSat) } } - return d.storeAddresses(wb, block, addresses) + return nil } func processedInTx(o []outpoint, btxID []byte) bool { @@ -646,10 +699,10 @@ func processedInTx(o []outpoint, btxID []byte) bool { return false } -func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, block *bchain.Block, addresses map[string][]outpoint) error { +func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error { for addrID, outpoints := range addresses { ba := []byte(addrID) - key := packAddressKey(ba, block.Height) + key := packAddressKey(ba, height) val := d.packOutpoints(outpoints) wb.PutCF(d.cfh[cfAddresses], key, val) }