diff --git a/api/types.go b/api/types.go index b605bd30..287e04af 100644 --- a/api/types.go +++ b/api/types.go @@ -1,5 +1,7 @@ package api +import "math/big" + type ScriptSig struct { Hex string `json:"hex"` Asm string `json:"asm,omitempty"` @@ -12,8 +14,8 @@ type Vin struct { N int `json:"n"` ScriptSig ScriptSig `json:"scriptSig"` Addr string `json:"addr"` - ValueSat int64 `json:"valueSat"` - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` } type ScriptPubKey struct { @@ -23,7 +25,8 @@ type ScriptPubKey struct { Type string `json:"type,omitempty"` } type Vout struct { - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` N int `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` SpentTxID string `json:"spentTxId,omitempty"` @@ -32,34 +35,30 @@ type Vout struct { } type Tx struct { - Txid string `json:"txid"` - Version int32 `json:"version,omitempty"` - Locktime uint32 `json:"locktime,omitempty"` - Vin []Vin `json:"vin"` - Vout []Vout `json:"vout"` - Blockhash string `json:"blockhash,omitempty"` - Blockheight int `json:"blockheight"` - Confirmations uint32 `json:"confirmations"` - Time int64 `json:"time,omitempty"` - Blocktime int64 `json:"blocktime"` - ValueOut float64 `json:"valueOut"` - Size int `json:"size,omitempty"` - ValueIn float64 `json:"valueIn"` - Fees float64 `json:"fees"` - WithSpends bool `json:"withSpends,omitempty"` + Txid string `json:"txid"` + Version int32 `json:"version,omitempty"` + Locktime uint32 `json:"locktime,omitempty"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` + Blockhash string `json:"blockhash,omitempty"` + Blockheight int `json:"blockheight"` + Confirmations uint32 `json:"confirmations"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime"` + ValueOut string `json:"valueOut"` + Size int `json:"size,omitempty"` + ValueIn string `json:"valueIn"` + Fees string `json:"fees"` + WithSpends bool `json:"withSpends,omitempty"` } type Address struct { - AddrStr string `json:"addrStr"` - Balance float64 `json:"balance"` - BalanceSat int64 `json:"balanceSat"` - TotalReceived float64 `json:"totalReceived"` - TotalReceivedSat int64 `json:"totalReceivedSat"` - TotalSent float64 `json:"totalSent"` - TotalSentSat int64 `json:"totalSentSat"` - UnconfirmedBalance float64 `json:"unconfirmedBalance"` - UnconfirmedBalanceSat int64 `json:"unconfirmedBalanceSat"` - UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` - TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"transactions"` + AddrStr string `json:"addrStr"` + Balance string `json:"balance"` + TotalReceived string `json:"totalReceived"` + TotalSent string `json:"totalSent"` + UnconfirmedBalance string `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*Tx `json:"transactions"` } diff --git a/api/worker.go b/api/worker.go index 4ad421d3..01064560 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "math/big" "github.com/golang/glog" ) @@ -44,7 +45,7 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) return nil, err } } - var valIn, valOut, fees float64 + var valInSat, valOutSat, feesSat big.Int vins := make([]Vin, len(bchainTx.Vin)) for i := range bchainTx.Vin { bchainVin := &bchainTx.Vin[i] @@ -61,9 +62,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) } if len(otx.Vout) > int(vin.Vout) { vout := &otx.Vout[vin.Vout] - vin.Value = vout.Value - valIn += vout.Value - vin.ValueSat = int64(vout.Value*1E8 + 0.5) + vin.ValueSat = vout.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) + valInSat.Add(&valInSat, &vout.ValueSat) if vout.Address != nil { a := vout.Address.String() vin.Addr = a @@ -76,8 +77,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) bchainVout := &bchainTx.Vout[i] vout := &vouts[i] vout.N = i - vout.Value = bchainVout.Value - valOut += bchainVout.Value + vout.ValueSat = bchainVout.ValueSat + vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) + valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses if spendingTx { @@ -85,9 +87,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) } } // for coinbase transactions valIn is 0 - fees = valIn - valOut - if fees < 0 { - fees = 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) } // for now do not return size, we would have to compute vsize of segwit transactions // size:=len(bchainTx.Hex) / 2 @@ -96,13 +98,13 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) Blockheight: int(height), Blocktime: bchainTx.Blocktime, Confirmations: bchainTx.Confirmations, - Fees: fees, + Fees: w.chainParser.AmountToDecimalString(&feesSat), Locktime: bchainTx.LockTime, WithSpends: spendingTx, Time: bchainTx.Time, Txid: bchainTx.Txid, - ValueIn: valIn, - ValueOut: valOut, + ValueIn: w.chainParser.AmountToDecimalString(&valInSat), + ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), Version: bchainTx.Version, Vin: vins, Vout: vouts, @@ -131,26 +133,26 @@ func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return txids, nil } -func (t *Tx) getAddrVoutValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVoutValue(addrID string) *big.Int { + var val big.Int for _, vout := range t.Vout { for _, a := range vout.ScriptPubKey.Addresses { if a == addrID { - val += vout.Value + val.Add(&val, &vout.ValueSat) } } } - return val + return &val } -func (t *Tx) getAddrVinValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVinValue(addrID string) *big.Int { + var val big.Int for _, vin := range t.Vin { if vin.Addr == addrID { - val += vin.Value + val.Add(&val, &vin.ValueSat) } } - return val + return &val } // UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions @@ -181,6 +183,7 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { if err != nil { return nil, err } + txm = UniqueTxidsInReverse(txm) bestheight, _, err := w.db.GetBestBlock() if err != nil { return nil, err @@ -191,14 +194,14 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { } txs := make([]*Tx, len(txm)+lc) txi := 0 - var uBal, bal, totRecv, totSent float64 + var uBalSat, balSat, totRecvSat, totSentSat big.Int for _, tx := range txm { tx, err := w.GetTransaction(tx, bestheight, false) // mempool transaction may fail if err != nil { glog.Error("GetTransaction ", tx, ": ", err) } else { - uBal = tx.getAddrVoutValue(addrID) - tx.getAddrVinValue(addrID) + uBalSat.Sub(tx.getAddrVoutValue(addrID), tx.getAddrVinValue(addrID)) txs[txi] = tx txi++ } @@ -216,26 +219,23 @@ func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { if err != nil { return nil, err } else { - totRecv += tx.getAddrVoutValue(addrID) - totSent += tx.getAddrVinValue(addrID) + totRecvSat.Add(&totRecvSat, tx.getAddrVoutValue(addrID)) + totSentSat.Add(&totSentSat, tx.getAddrVinValue(addrID)) if i >= from && i < to { txs[txi] = tx txi++ } } } - bal = totRecv - totSent + balSat.Sub(&totRecvSat, &totSentSat) r := &Address{ AddrStr: addrID, - Balance: bal, - BalanceSat: int64(bal*1E8 + 0.5), - TotalReceived: totRecv, - TotalReceivedSat: int64(totRecv*1E8 + 0.5), - TotalSent: totSent, - TotalSentSat: int64(totSent*1E8 + 0.5), + Balance: w.chainParser.AmountToDecimalString(&balSat), + TotalReceived: w.chainParser.AmountToDecimalString(&totRecvSat), + TotalSent: w.chainParser.AmountToDecimalString(&totSentSat), Transactions: txs[:txi], TxApperances: len(txc), - UnconfirmedBalance: uBal, + UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), UnconfirmedTxApperances: len(txm), } glog.Info(addrID, " finished") diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 8f7bc08a..172a8180 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -3,6 +3,8 @@ package bchain import ( "encoding/hex" "encoding/json" + "math/big" + "strings" "github.com/gogo/protobuf/proto" "github.com/juju/errors" @@ -14,6 +16,7 @@ type AddressFactoryFunc func(string) (Address, error) type BaseParser struct { AddressFactory AddressFactoryFunc BlockAddressesToKeep int + AmountDecimalPoint int } // AddressToOutputScript converts address to ScriptPubKey - currently not implemented @@ -36,7 +39,52 @@ func (p *BaseParser) ParseTx(b []byte) (*Tx, error) { return nil, errors.New("ParseTx: not implemented") } -// ParseTxFromJson parses JSON message containing transaction and returs Tx struct +const zeros = "0000000000000000000000000000000000000000" + +// AmountToBigInt converts amount in json.Number (string) to big.Int +// it uses string operations to avoid problems with rounding +func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { + var r big.Int + s := string(n) + i := strings.IndexByte(s, '.') + if i == -1 { + s = s + zeros[:p.AmountDecimalPoint] + } else { + z := p.AmountDecimalPoint - len(s) + i + 1 + if z > 0 { + s = s[:i] + s[i+1:] + zeros[:z] + } else { + s = s[:i] + s[i+1:len(s)+z] + } + } + if _, ok := r.SetString(s, 10); !ok { + return r, errors.New("AmountToBigInt: failed to convert") + } + return r, nil +} + +// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place +func (p *BaseParser) AmountToDecimalString(a *big.Int) string { + n := a.String() + var s string + if n[0] == '-' { + n = n[1:] + s = "-" + } + if len(n) <= p.AmountDecimalPoint { + n = zeros[:p.AmountDecimalPoint-len(n)+1] + n + } + i := len(n) - p.AmountDecimalPoint + ad := strings.TrimRight(n[i:], "0") + if len(ad) > 0 { + n = n[:i] + "." + ad + } else { + n = n[:i] + } + return s + n +} + +// ParseTxFromJson parses JSON message containing transaction and returns Tx struct func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { var tx Tx err := json.Unmarshal(msg, &tx) @@ -44,13 +92,20 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { return nil, err } - for i, vout := range tx.Vout { + for i := range tx.Vout { + vout := &tx.Vout[i] + // convert vout.JsonValue to big.Int and clear it, it is only temporary value used for unmarshal + vout.ValueSat, err = p.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } + vout.JsonValue = "" if len(vout.ScriptPubKey.Addresses) == 1 { a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) if err != nil { return nil, err } - tx.Vout[i].Address = a + vout.Address = a } } @@ -127,7 +182,7 @@ func (p *BaseParser) PackTx(tx *Tx, height uint32, blockTime int64) ([]byte, err Addresses: vo.ScriptPubKey.Addresses, N: vo.N, ScriptPubKeyHex: hex, - Value: vo.Value, + ValueSat: vo.ValueSat.Bytes(), } } pt := &ProtoTransaction{ @@ -176,13 +231,15 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } vout := make([]Vout, len(pt.Vout)) for i, pto := range pt.Vout { + var vs big.Int + vs.SetBytes(pto.ValueSat) vout[i] = Vout{ N: pto.N, ScriptPubKey: ScriptPubKey{ Addresses: pto.Addresses, Hex: hex.EncodeToString(pto.ScriptPubKeyHex), }, - Value: pto.Value, + ValueSat: vs, } if len(pto.Addresses) == 1 { a, err := p.AddressFactory(pto.Addresses[0]) diff --git a/bchain/baseparser_test.go b/bchain/baseparser_test.go new file mode 100644 index 00000000..cfb20a68 --- /dev/null +++ b/bchain/baseparser_test.go @@ -0,0 +1,56 @@ +package bchain + +import ( + "encoding/json" + "math/big" + "testing" +) + +func NewBaseParser(adp int) *BaseParser { + return &BaseParser{ + AmountDecimalPoint: adp, + } +} + +var amounts = []struct { + a *big.Int + s string + adp int + alternative string +}{ + {big.NewInt(123456789), "1.23456789", 8, "!"}, + {big.NewInt(2), "0.00000002", 8, "!"}, + {big.NewInt(300000000), "3", 8, "!"}, + {big.NewInt(498700000), "4.987", 8, "!"}, + {big.NewInt(567890), "0.00000000000056789", 18, "!"}, + {big.NewInt(-100000000), "-1", 8, "!"}, + {big.NewInt(-8), "-0.00000008", 8, "!"}, + {big.NewInt(-89012345678), "-890.12345678", 8, "!"}, + {big.NewInt(-12345), "-0.00012345", 8, "!"}, + {big.NewInt(12345678), "0.123456789012", 8, "0.12345678"}, // test of truncation of too many decimal places +} + +func TestBaseParser_AmountToDecimalString(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + if got := NewBaseParser(tt.adp).AmountToDecimalString(tt.a); got != tt.s && got != tt.alternative { + t.Errorf("BaseParser.AmountToDecimalString() = %v, want %v", got, tt.s) + } + }) + } +} + +func TestBaseParser_AmountToBigInt(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + got, err := NewBaseParser(tt.adp).AmountToBigInt(json.Number(tt.s)) + if err != nil { + t.Errorf("BaseParser.AmountToBigInt() error = %v", err) + return + } + if got.Cmp(tt.a) != 0 { + t.Errorf("BaseParser.AmountToBigInt() = %v, want %v", got, tt.a) + } + }) + } +} diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 95284d84..02286ca6 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -49,6 +49,7 @@ func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) (*BCashParser BaseParser: &bchain.BaseParser{ AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, format) }, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, Params: params, OutputScriptToAddressesFunc: outputScriptToAddresses, diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index 3e1cb4c1..d6040b82 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -7,6 +7,7 @@ import ( "blockbook/bchain/coins/btc" "bytes" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -159,8 +160,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ @@ -190,8 +191,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ @@ -201,8 +202,8 @@ func init() { Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 68af837a..5dd9b141 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -5,6 +5,7 @@ import ( "blockbook/bchain/coins/btc" "encoding/hex" "encoding/json" + "math/big" "github.com/cpacia/bchutil" "github.com/golang/glog" @@ -132,7 +133,7 @@ func (b *BCashRPC) GetBlockFull(hash string) (*bchain.Block, error) { } // EstimateSmartFee returns fee estimation. -func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := btc.ResEstimateSmartFee{} @@ -141,13 +142,18 @@ func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, err // conservative param is omitted err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } func isErrBlockNotFound(err *bchain.RPCError) bool { diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 7a7a7241..ddd1ffd2 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/big" "reflect" "time" @@ -175,12 +176,12 @@ func (c *blockChainWithMetrics) GetTransactionForMempool(txid string) (v *bchain return c.b.GetTransactionForMempool(txid) } -func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateSmartFee", s, err) }(time.Now()) return c.b.EstimateSmartFee(blocks, conservative) } -func (c *blockChainWithMetrics) EstimateFee(blocks int) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateFee(blocks int) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateFee", s, err) }(time.Now()) return c.b.EstimateFee(blocks) } diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 861c7f31..767e8d08 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" vlq "github.com/bsm/go-vlq" "github.com/btcsuite/btcd/blockchain" @@ -30,6 +31,7 @@ func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser &bchain.BaseParser{ AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, params, outputScriptToAddresses, @@ -123,8 +125,10 @@ func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.T // missing: Asm, // missing: Type, } + var vs big.Int + vs.SetInt64(out.Value) vout[i] = bchain.Vout{ - Value: float64(out.Value) / 1E8, + ValueSat: vs, N: uint32(i), ScriptPubKey: s, } diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index d605f45b..79d02816 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -5,6 +5,7 @@ package btc import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -152,8 +153,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ @@ -183,8 +184,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ @@ -194,8 +195,8 @@ func init() { Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index e0176d16..a6305106 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -8,6 +8,7 @@ import ( "encoding/json" "io" "io/ioutil" + "math/big" "net" "net/http" "time" @@ -36,19 +37,21 @@ type BitcoinRPC struct { } type Configuration struct { - CoinName string `json:"coin_name"` - CoinShortcut string `json:"coin_shortcut"` - RPCURL string `json:"rpc_url"` - RPCUser string `json:"rpc_user"` - RPCPass string `json:"rpc_pass"` - RPCTimeout int `json:"rpc_timeout"` - Parse bool `json:"parse"` - MessageQueueBinding string `json:"message_queue_binding"` - Subversion string `json:"subversion"` - BlockAddressesToKeep int `json:"block_addresses_to_keep"` - MempoolWorkers int `json:"mempool_workers"` - MempoolSubWorkers int `json:"mempool_sub_workers"` - AddressFormat string `json:"address_format"` + CoinName string `json:"coin_name"` + CoinShortcut string `json:"coin_shortcut"` + RPCURL string `json:"rpc_url"` + RPCUser string `json:"rpc_user"` + RPCPass string `json:"rpc_pass"` + RPCTimeout int `json:"rpc_timeout"` + Parse bool `json:"parse"` + MessageQueueBinding string `json:"message_queue_binding"` + Subversion string `json:"subversion"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` + MempoolWorkers int `json:"mempool_workers"` + MempoolSubWorkers int `json:"mempool_sub_workers"` + AddressFormat string `json:"address_format"` + SupportsEstimateFee bool `json:"supports_estimate_fee"` + SupportsEstimateSmartFee bool `json:"supports_estimate_smart_fee"` } // NewBitcoinRPC returns new BitcoinRPC instance. @@ -70,6 +73,9 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT if c.MempoolSubWorkers < 1 { c.MempoolSubWorkers = 1 } + // btc supports both calls, other coins overriding BitcoinRPC can change this + c.SupportsEstimateFee = true + c.SupportsEstimateSmartFee = true transport := &http.Transport{ Dial: (&net.Dialer{KeepAlive: 600 * time.Second}).Dial, @@ -302,8 +308,8 @@ type CmdEstimateSmartFee struct { type ResEstimateSmartFee struct { Error *bchain.RPCError `json:"error"` Result struct { - Feerate float64 `json:"feerate"` - Blocks int `json:"blocks"` + Feerate json.Number `json:"feerate"` + Blocks int `json:"blocks"` } `json:"result"` } @@ -318,7 +324,7 @@ type CmdEstimateFee struct { type ResEstimateFee struct { Error *bchain.RPCError `json:"error"` - Result float64 `json:"result"` + Result json.Number `json:"result"` } // sendrawtransaction @@ -617,7 +623,12 @@ func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]string, error) { } // EstimateSmartFee returns fee estimation. -func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + // use EstimateFee if EstimateSmartFee is not supported + if !b.ChainConfig.SupportsEstimateSmartFee && b.ChainConfig.SupportsEstimateFee { + return b.EstimateFee(blocks) + } + glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := ResEstimateSmartFee{} @@ -630,17 +641,27 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, e } err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } // EstimateFee returns fee estimation. -func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { +func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { + // use EstimateSmartFee if EstimateFee is not supported + if !b.ChainConfig.SupportsEstimateFee && b.ChainConfig.SupportsEstimateSmartFee { + return b.EstimateSmartFee(blocks, true) + } + glog.V(1).Info("rpc: estimatefee ", blocks) res := ResEstimateFee{} @@ -648,13 +669,18 @@ func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { req.Params.Blocks = blocks err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result, nil + r, err = b.Parser.AmountToBigInt(res.Result) + if err != nil { + return r, err + } + return r, nil } // SendRawTransaction sends raw transaction. @@ -685,13 +711,20 @@ func (b *BitcoinRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) Params: []string{txid}, } err := b.Call(&req, &res) - if err != nil { return nil, err } if res.Error != nil { return nil, res.Error } + res.Result.FeeSat, err = b.Parser.AmountToBigInt(res.Result.Fee) + if err != nil { + return nil, err + } + res.Result.ModifiedFeeSat, err = b.Parser.AmountToBigInt(res.Result.ModifiedFee) + if err != nil { + return nil, err + } return res.Result, nil } diff --git a/bchain/coins/dash/dashrpc.go b/bchain/coins/dash/dashrpc.go index 4c884f08..dec0ee49 100644 --- a/bchain/coins/dash/dashrpc.go +++ b/bchain/coins/dash/dashrpc.go @@ -24,6 +24,7 @@ func NewDashRPC(config json.RawMessage, pushHandler func(bchain.NotificationType b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -53,8 +54,3 @@ func (b *DashRPC) Initialize() error { return nil } - -// EstimateSmartFee returns fee estimation. -func (b *DashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/dogecoin/dogecoinparser_test.go b/bchain/coins/dogecoin/dogecoinparser_test.go index b8ea585f..0b736c51 100644 --- a/bchain/coins/dogecoin/dogecoinparser_test.go +++ b/bchain/coins/dogecoin/dogecoinparser_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "path/filepath" "reflect" "testing" @@ -111,8 +112,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 27478.75452951, - N: 0, + ValueSat: *big.NewInt(2747875452951), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914eef21768a546590993e313c7f3dfadf6a6efa1e888ac", Addresses: []string{ @@ -122,8 +123,8 @@ func init() { Address: addr1, }, { - Value: 74.20567469, - N: 1, + ValueSat: *big.NewInt(7420567469), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e0fee2ea29dd9c6c759d8341bd0da4c4f738cced88ac", Addresses: []string{ @@ -153,8 +154,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 59890867.89818935, - N: 0, + ValueSat: *big.NewInt(5989086789818935), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149355c01ed20057eac9fe0bbf8b07d87e62fe712d88ac", Addresses: []string{ @@ -164,8 +165,8 @@ func init() { Address: addr3, }, { - Value: 9999998.90000000, - N: 1, + ValueSat: *big.NewInt(999999890000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9145b4f2511c94e4fcaa8f8835b2458f8cb6542ca7688ac", Addresses: []string{ diff --git a/bchain/coins/dogecoin/dogecoinrpc.go b/bchain/coins/dogecoin/dogecoinrpc.go index d0ebb31c..1fc21618 100644 --- a/bchain/coins/dogecoin/dogecoinrpc.go +++ b/bchain/coins/dogecoin/dogecoinrpc.go @@ -24,6 +24,7 @@ func NewDogecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *DogecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *DogecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index fe361258..82d08153 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -22,7 +22,11 @@ type EthereumParser struct { // NewEthereumParser returns new EthereumParser instance func NewEthereumParser() *EthereumParser { - return &EthereumParser{&bchain.BaseParser{AddressFactory: bchain.NewBaseAddress}} + return &EthereumParser{&bchain.BaseParser{ + AddressFactory: bchain.NewBaseAddress, + BlockAddressesToKeep: 0, + AmountDecimalPoint: 18, + }} } type rpcTransaction struct { @@ -86,6 +90,10 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } tx.BlockHash = bh h := hex.EncodeToString(b) + vs, err := hexutil.DecodeBig(tx.Value) + if err != nil { + return nil, err + } return &bchain.Tx{ Blocktime: blocktime, Confirmations: confirmations, @@ -105,8 +113,8 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma }, Vout: []bchain.Vout{ { - N: 0, // there is always up to one To address - // Value - cannot be set, it does not fit precisely to float64 + N: 0, // there is always up to one To address + ValueSat: *vs, ScriptPubKey: bchain.ScriptPubKey{ // Hex Addresses: ta, @@ -286,9 +294,3 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) { func (p *EthereumParser) IsUTXOChain() bool { return false } - -// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column -// do not use the blockaddresses for eth -func (p *EthereumParser) KeepBlockAddresses() int { - return 0 -} diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index c1f4620f..40069a55 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -5,6 +5,7 @@ package eth import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -63,6 +64,68 @@ func TestEthParser_GetAddrIDFromAddress(t *testing.T) { } } +var ( + testTx1, testTx2 bchain.Tx + testTxPacked1 = "08aebf0a1205012a05f20018a0f73622081234567890abcdef2a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f" + testTxPacked2 = "08ece40212050430e234001888a4012201213220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915" +) + +func init() { + var ( + addr1, addr2 bchain.Address + err error + ) + addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") + if err == nil { + addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") + } + if err != nil { + panic(err) + } + + testTx1 = bchain.Tx{ + Blocktime: 1521515026, + Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307831323334353637383930616263646566222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", + Time: 1521515026, + Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + Vin: []bchain.Vin{ + { + Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(1311768467294899695), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, + }, + Address: addr1, + }, + }, + } + + testTx2 = bchain.Tx{ + Blocktime: 1521533434, + Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a2230783231222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", + Time: 1521533434, + Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", + Vin: []bchain.Vin{ + { + Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(33), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, + }, + Address: addr2, + }, + }, + } +} + func TestEthereumParser_PackTx(t *testing.T) { type args struct { tx *bchain.Tx @@ -77,61 +140,27 @@ func TestEthereumParser_PackTx(t *testing.T) { wantErr bool }{ { - name: "with 0x prefix", + name: "1", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - }, - }, - }, + tx: &testTx1, height: 2870000, blockTime: 1521515026, }, - want: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f", + want: testTxPacked1, }, { - name: "without 0x prefix", + name: "2", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - }, - }, - }, + tx: &testTx2, height: 2871048, blockTime: 1521533434, }, - want: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915", + want: testTxPacked2, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime) if (err != nil) != tt.wantErr { t.Errorf("EthereumParser.PackTx() error = %v, wantErr %v", err, tt.wantErr) @@ -146,18 +175,6 @@ func TestEthereumParser_PackTx(t *testing.T) { } func TestEthereumParser_UnpackTx(t *testing.T) { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") - if err == nil { - addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") - } - if err != nil { - panic(err) - } - type args struct { hex string } @@ -170,57 +187,21 @@ func TestEthereumParser_UnpackTx(t *testing.T) { wantErr bool }{ { - name: "1", - args: args{hex: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f"}, - want: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - Address: addr1, - }, - }, - }, + name: "1", + args: args{hex: testTxPacked1}, + want: &testTx1, want1: 2870000, }, { - name: "1", - args: args{hex: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915"}, - want: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - Address: addr2, - }, - }, - }, + name: "2", + args: args{hex: testTxPacked2}, + want: &testTx2, want1: 2871048, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() b, err := hex.DecodeString(tt.args.hex) if err != nil { panic(err) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 1285c354..0a19e10b 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -479,12 +479,12 @@ func (b *EthereumRPC) GetMempool() ([]string, error) { } // EstimateFee returns fee estimation. -func (b *EthereumRPC) EstimateFee(blocks int) (float64, error) { +func (b *EthereumRPC) EstimateFee(blocks int) (big.Int, error) { return b.EstimateSmartFee(blocks, true) } // EstimateSmartFee returns fee estimation. -func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() // TODO - what parameters of msg to use to get better estimate, maybe more data from the wallet are needed @@ -493,10 +493,12 @@ func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, To: &a, } g, err := b.client.EstimateGas(ctx, msg) + var r big.Int if err != nil { - return 0, err + return r, err } - return float64(g), nil + r.SetUint64(g) + return r, nil } // SendRawTransaction sends raw transaction. diff --git a/bchain/coins/litecoin/litecoinparser_test.go b/bchain/coins/litecoin/litecoinparser_test.go index 11c55f1e..a76faf63 100644 --- a/bchain/coins/litecoin/litecoinparser_test.go +++ b/bchain/coins/litecoin/litecoinparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -173,8 +174,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 12.52000000, - N: 0, + ValueSat: *big.NewInt(1252000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9141ae882e788091732da6910595314447c9e38bd8d88ac", Addresses: []string{ @@ -184,8 +185,8 @@ func init() { Address: addr1, }, { - Value: 0.01000487, - N: 1, + ValueSat: *big.NewInt(1000487), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9146b474cbf0f6004329b630bdd4798f2c23d1751b688ac", Addresses: []string{ diff --git a/bchain/coins/litecoin/litecoinrpc.go b/bchain/coins/litecoin/litecoinrpc.go index 0d2ad15f..e7345251 100644 --- a/bchain/coins/litecoin/litecoinrpc.go +++ b/bchain/coins/litecoin/litecoinrpc.go @@ -24,6 +24,7 @@ func NewLitecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *LitecoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *LitecoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/namecoin/namecoinrpc.go b/bchain/coins/namecoin/namecoinrpc.go index faa14fd3..6ab0b0aa 100644 --- a/bchain/coins/namecoin/namecoinrpc.go +++ b/bchain/coins/namecoin/namecoinrpc.go @@ -24,6 +24,7 @@ func NewNamecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *NamecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *NamecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/vertcoin/vertcoinparser_test.go b/bchain/coins/vertcoin/vertcoinparser_test.go index 558ae51f..0cbf01d5 100644 --- a/bchain/coins/vertcoin/vertcoinparser_test.go +++ b/bchain/coins/vertcoin/vertcoinparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -105,8 +106,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.10781192, - N: 0, + ValueSat: *big.NewInt(10781192), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a91499b16da88a7e29b913b6131df2644d6d06cb331b88ac", Addresses: []string{ @@ -116,8 +117,8 @@ func init() { Address: addr1, }, { - Value: 0.5000000, - N: 1, + ValueSat: *big.NewInt(50000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a91446eb90e002f137f05385896c882fe000cc2e967f87", Addresses: []string{ diff --git a/bchain/coins/vertcoin/vertcoinrpc.go b/bchain/coins/vertcoin/vertcoinrpc.go index 878b9682..f282dc78 100644 --- a/bchain/coins/vertcoin/vertcoinrpc.go +++ b/bchain/coins/vertcoin/vertcoinrpc.go @@ -24,6 +24,7 @@ func NewVertcoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *VertcoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *VertcoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index eacf8bbd..dc2d84d9 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -20,12 +20,13 @@ type ZCashParser struct { *bchain.BaseParser } -// NewZCAshParser returns new ZCAshParser instance +// NewZCashParser returns new ZCashParser instance func NewZCashParser(c *btc.Configuration) *ZCashParser { return &ZCashParser{ &bchain.BaseParser{ AddressFactory: bchain.NewBaseAddress, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, } } diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index 0bb61fe1..b1bfd773 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -6,6 +6,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -13,8 +14,8 @@ import ( var ( testTx1, testTx2 bchain.Tx - testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a4b091e6c90cd3ebc664010001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" - testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a4b09257b2170264d504010001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a4b099a9999999999b93f10011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" + testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a490a05043c1aec8e10001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" + testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a490a050184a7a72310001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a470a0398968010011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" ) func init() { @@ -51,8 +52,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 181.88266638, - N: 0, + ValueSat: *big.NewInt(18188266638), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac", Addresses: []string{ @@ -82,8 +83,8 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 65.20547107, - N: 0, + ValueSat: *big.NewInt(6520547107), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914826f87806ddd4643730be99b41c98acc379e83db88ac", Addresses: []string{ @@ -93,8 +94,8 @@ func init() { Address: addr2, }, { - Value: .1, - N: 1, + ValueSat: *big.NewInt(10000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e395634b7684289285926d4c64db395b783720ec88ac", Addresses: []string{ diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 495fa7af..b2c45fe5 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -22,6 +22,7 @@ func NewZCashRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp BitcoinRPC: b.(*btc.BitcoinRPC), } z.RPCMarshaler = btc.JSONMarshalerV1{} + z.ChainConfig.SupportsEstimateSmartFee = false return z, nil } @@ -114,14 +115,6 @@ func (z *ZCashRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { return z.GetTransaction(txid) } -// EstimateSmartFee returns fee estimation. -func (z *ZCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - glog.V(1).Info("rpc: estimatesmartfee") - - // return z.estimateFee(blocks) - return z.EstimateFee(blocks) -} - // GetMempoolEntry returns mempool data for given transaction func (z *ZCashRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { return nil, errors.New("GetMempoolEntry: not implemented") diff --git a/bchain/tests/rpc/data.go b/bchain/tests/rpc/data.go index 7d50460a..68d69acf 100644 --- a/bchain/tests/rpc/data.go +++ b/bchain/tests/rpc/data.go @@ -1,8 +1,7 @@ -// +build integration - package rpc import ( + "blockbook/bchain" "encoding/json" "errors" "fmt" @@ -83,7 +82,7 @@ func LoadRPCConfig(coin string) (json.RawMessage, error) { return json.RawMessage(fmt.Sprintf(t, coin, c.URL, c.User, c.Pass)), nil } -func LoadTestData(coin string) (*TestData, error) { +func LoadTestData(coin string, parser bchain.BlockChainParser) (*TestData, error) { b, err := readDataFile(".", "bchain/tests/rpc/testdata", coin+".json") if err != nil { return nil, err @@ -93,5 +92,16 @@ func LoadTestData(coin string) (*TestData, error) { if err != nil { return nil, err } + // convert amounts in test json to bit.Int and clear the temporary JsonValue + for _, tx := range v.TxDetails { + for i := range tx.Vout { + vout := &tx.Vout[i] + vout.ValueSat, err = parser.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } + vout.JsonValue = "" + } + } return &v, nil } diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go index 4386baa1..20483573 100644 --- a/bchain/tests/rpc/rpc.go +++ b/bchain/tests/rpc/rpc.go @@ -1,5 +1,3 @@ -// +build integration - package rpc import ( @@ -9,6 +7,7 @@ import ( "net" "reflect" "testing" + "time" "github.com/deckarep/golang-set" ) @@ -56,7 +55,7 @@ func NewTest(coin string, factory TestChainFactoryFunc) (*Test, error) { return nil, err } } else { - td, err = LoadTestData(coin) + td, err = LoadTestData(coin, cli.GetChainParser()) if err != nil { return nil, err } @@ -162,7 +161,7 @@ func (rt *Test) TestGetTransaction(t *testing.T) { got.Confirmations = 0 if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransaction() got %v, want %v", got, want) + t.Errorf("GetTransaction() got %+v, want %+v", got, want) } } } @@ -181,7 +180,7 @@ func (rt *Test) TestGetTransactionForMempool(t *testing.T) { // transactions parsed from JSON may contain additional data got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransactionForMempool() got %v, want %v", got, want) + t.Errorf("GetTransactionForMempool() got %+v, want %+v", got, want) } } } @@ -329,8 +328,8 @@ func (rt *Test) TestGetMempoolEntry(t *testing.T) { if e.Size <= 0 { t.Errorf("GetMempoolEntry() got zero or negative size %d", e.Size) } - if e.Fee <= 0 { - t.Errorf("GetMempoolEntry() got zero or negative fee %f", e.Fee) + if e.FeeSat.Sign() != 1 { + t.Errorf("GetMempoolEntry() got zero or negative fee %v", e.FeeSat.String()) } // done @@ -347,8 +346,11 @@ func (rt *Test) TestEstimateSmartFee(t *testing.T) { if err != nil { t.Error(err) } - if fee != -1 && fee < 0 { - t.Errorf("EstimateSmartFee() returned unexpected fee rate: %f", fee) + if fee.Sign() == -1 { + sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateSmartFee() returned unexpected fee rate: %v", sf) + } } } } @@ -361,8 +363,11 @@ func (rt *Test) TestEstimateFee(t *testing.T) { if err != nil { t.Error(err) } - if fee != -1 && fee < 0 { - t.Errorf("EstimateFee() returned unexpected fee rate: %f", fee) + if fee.Sign() == -1 { + sf := rt.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateFee() returned unexpected fee rate: %v", sf) + } } } } @@ -385,6 +390,7 @@ func (rt *Test) TestGetBestBlockHash(t *testing.T) { t.Fatal(err) } if hash != hh { + time.Sleep(time.Millisecond * 100) continue } diff --git a/bchain/tests/rpc/testdata/Bcash.json b/bchain/tests/rpc/testdata/Bcash.json index f72df31f..9b1f647b 100644 --- a/bchain/tests/rpc/testdata/Bcash.json +++ b/bchain/tests/rpc/testdata/Bcash.json @@ -70,6 +70,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "fe496933e0a3582ef020bd35c38fe8244a80fa7c63e7607f6f9ccb0806f419e4", @@ -175,6 +176,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "52f37874fd5a2497c5d84619ab415ca17ebb1d49d559f0d468869f70537354b9", diff --git a/bchain/tests/rpc/testdata/Bcash_Testnet.json b/bchain/tests/rpc/testdata/Bcash_Testnet.json index 39eba5e7..c33176d3 100644 --- a/bchain/tests/rpc/testdata/Bcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Bcash_Testnet.json @@ -12,6 +12,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "coinbase": "03fbf212055374617368", @@ -34,6 +35,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "txid": "58ff2450a71b3c228d17d04ec8edcaae452bc97d5ccc4591e2741fd8b031d221", diff --git a/bchain/tests/rpc/testdata/Bitcoin.json b/bchain/tests/rpc/testdata/Bitcoin.json index ba0467cc..5cf34ec8 100644 --- a/bchain/tests/rpc/testdata/Bitcoin.json +++ b/bchain/tests/rpc/testdata/Bitcoin.json @@ -120,6 +120,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 0, + "version": 1, "vin": [ { "txid": "83eff5bcc738d52e04528984cd5cf601b69e7df65b2b10ed2475c0072cdde14c", @@ -153,6 +154,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 529149, + "version": 2, "vin": [ { "txid": "413988d546516707f3b20ca9876e026a9472bd814662e42f37f4e57a15713da7", diff --git a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json index ece3c941..cab2c840 100644 --- a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json +++ b/bchain/tests/rpc/testdata/Bitcoin_Testnet.json @@ -43,6 +43,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "4097f5265397047bffe219a1bca65ea5726402c3d9aeecd577fa1c274fc1b8a4", @@ -84,6 +85,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "80437ba4e81bde8b584258f5adad0d08dea60f1e38026344442ad59a4ef797c9", diff --git a/bchain/tests/rpc/testdata/Dash.json b/bchain/tests/rpc/testdata/Dash.json index 150e7d51..f5ab09b6 100644 --- a/bchain/tests/rpc/testdata/Dash.json +++ b/bchain/tests/rpc/testdata/Dash.json @@ -27,6 +27,7 @@ "blocktime": 1530189699, "time": 1530189699, "locktime": 0, + "version": 1, "vin": [ { "txid": "a78824a88c089ea1344fe2a33c454024e6501e76cca2590515b390d85082bd23", diff --git a/bchain/tests/rpc/testdata/Dash_Testnet.json b/bchain/tests/rpc/testdata/Dash_Testnet.json index d86f0bee..64509772 100644 --- a/bchain/tests/rpc/testdata/Dash_Testnet.json +++ b/bchain/tests/rpc/testdata/Dash_Testnet.json @@ -31,6 +31,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "a41616c7585c98aeda98d6ff6766b15455e327c9472582b80289dab7597ad309", @@ -64,6 +65,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "187ae015e41dff766f5a18ae705f59db950a6729a06fa5fd04630c362a9aee27", diff --git a/bchain/tests/rpc/testdata/Ethereum_Testnet.json b/bchain/tests/rpc/testdata/Ethereum_Testnet.json index 82bbf3c7..b7cd76cb 100644 --- a/bchain/tests/rpc/testdata/Ethereum_Testnet.json +++ b/bchain/tests/rpc/testdata/Ethereum_Testnet.json @@ -16,38 +16,21 @@ "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0" ], "txDetails": { - "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d": { - "hex": "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - "txid": "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0": { + "hex": "7b226e6f6e6365223a223078333632306136222c226761735072696365223a223078313261303566323030222c22676173223a22307835323038222c22746f223a22307831623137626331326166623635643563346238316139666632613037366234326131396661616136222c2276616c7565223a223078646530623662336137363430303030222c22696e707574223a223078222c2268617368223a22307837663064313430333239393431663132306235623366633735316533306164656238376232616562626663653561646364303231363630346133346236636330222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838316237653038663635626466353634383630366338393939386139636338313634333937363437222c227472616e73616374696f6e496e646578223a22307862222c2276223a2230783162222c2272223a22307862666662323864633865373939383833366639356664616139396433626435666635346365663562313839636462383537333537666161326431616231393136222c2273223a22307833616462316365313264396664306538616662373064386639346534636538356137666639346465333966623636333139363638363435663464643138646432227d", + "txid": "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0", "blocktime": 1521515026, "time": 1521515026, "vin": [ { - "addresses": ["0xdacc9c61754a0c4616fc5323dc946e89eb272302"] + "addresses": ["0x81b7e08f65bdf5648606c89998a9cc8164397647"] } ], "vout": [ { + "value": 1, "scriptPubKey": { - "addresses": ["0x682b7903a11098cf770c7aef4aa02a85b3f3601a"] - } - } - ] - }, - "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3": { - "hex": "7b226e6f6e6365223a223078346566222c226761735072696365223a223078626134336237343030222c22676173223a2230783936653838222c22746f223a22307864303534323939633438326164356362333961336661383734373235663965326132306433343963222c2276616c7565223a22307830222c22696e707574223a2230783165626366613630303030303030303030303030303030303030303030303030633034383732383566313736663532306635636434363732306236383662366534366165636430373030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303031303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030313430303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030633461303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303530303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030353831303030303030303030303030303030303030303030303030633365313631343261306432366336616530623163656238643961373236653734653164613664663666656562653137653437306464626434656261663364623938333136386235316337623136363732613131623966306662383864623438323962653237346230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303034363336663665363230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303437323666373336313030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030222c2268617368223a22307831376565323335666330333539313535623235343139653065346336356439633530306466366537316538323838643665663032306430346363326632636233222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838623461353365353739303739343466633261653539623837373066393331623639326232373062222c227472616e73616374696f6e496e646578223a22307830222c2276223a2230783163222c2272223a22307866633739633836353638323039313030323134616339353662373930653066383935656130343135313438643233613239353632356564393761633936333534222c2273223a223078373232303833616331643764663662626162393939383537616163323561336665646265386130633561326266376364653835393738363266373862313937227d", - "txid": "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", - "blocktime": 1521515026, - "time": 1521515026, - "vin": [ - { - "addresses": ["0x8b4a53e57907944fc2ae59b8770f931b692b270b"] - } - ], - "vout": [ - { - "scriptPubKey": { - "addresses": ["0xd054299c482ad5cb39a3fa874725f9e2a20d349c"] + "addresses": ["0x1b17bc12afb65d5c4b81a9ff2a076b42a19faaa6"] } } ] diff --git a/bchain/tests/rpc/testdata/Namecoin.json b/bchain/tests/rpc/testdata/Namecoin.json index 4e1193cd..5a56c972 100644 --- a/bchain/tests/rpc/testdata/Namecoin.json +++ b/bchain/tests/rpc/testdata/Namecoin.json @@ -39,6 +39,7 @@ "blocktime": 1530003649, "time": 1530003649, "locktime": 404678, + "version": 1, "vin": [ { "txid": "223a4ff7a613173ffbc67f9efffe1b0c3fd5ba2471935a44cc81814038272901", diff --git a/bchain/tests/rpc/testdata/Vertcoin.json b/bchain/tests/rpc/testdata/Vertcoin.json index 021cdc2a..d5c5383e 100644 --- a/bchain/tests/rpc/testdata/Vertcoin.json +++ b/bchain/tests/rpc/testdata/Vertcoin.json @@ -14,6 +14,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952232, + "version": 2, "vin": [ { "txid": "8ec70404bdd16559c589736863ee9d9f69431ba4e6b2ab3b1641b3f9f3821624", @@ -47,6 +48,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952233, + "version": 1, "vin": [ { "txid": "5765db7dd62cddada154b2c95505e298818a308cc1ce1e28d6e8565301cb8d74", diff --git a/bchain/tests/rpc/testdata/Zcash.json b/bchain/tests/rpc/testdata/Zcash.json index 6b620537..346d9f96 100644 --- a/bchain/tests/rpc/testdata/Zcash.json +++ b/bchain/tests/rpc/testdata/Zcash.json @@ -22,6 +22,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 349399, + "version": 3, "vin": [ { "txid": "9c9faac29b0fa1b0e683727f2973bfcb87e4baadd07e9c997b431a31713cb30c", @@ -57,6 +58,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 0, + "version": 3, "vin": [ { "txid": "4440c8e9d5b57da7ca0fb3f62ec13f392267aeb797c123bb01e850adf8573dd0", diff --git a/bchain/tests/rpc/testdata/Zcash_Testnet.json b/bchain/tests/rpc/testdata/Zcash_Testnet.json index aca84131..5d78989c 100644 --- a/bchain/tests/rpc/testdata/Zcash_Testnet.json +++ b/bchain/tests/rpc/testdata/Zcash_Testnet.json @@ -13,6 +13,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251028, + "version": 3, "vin": [ { "txid": "19a1d013b898239e9a2943faa07f8716b9be168bc8e001daf3625f535fde1a60", @@ -48,6 +49,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251090, + "version": 3, "vin": [ { "txid": "9acab5f13cf94074e75f5686b59fccd938f54b5f20ddddfcb6077c679a13c0ea", diff --git a/bchain/tx.pb.go b/bchain/tx.pb.go index dc5a0952..2115993a 100644 --- a/bchain/tx.pb.go +++ b/bchain/tx.pb.go @@ -148,7 +148,7 @@ func (m *ProtoTransaction_VinType) GetAddresses() []string { } type ProtoTransaction_VoutType struct { - Value float64 `protobuf:"fixed64,1,opt,name=Value" json:"Value,omitempty"` + ValueSat []byte `protobuf:"bytes,1,opt,name=ValueSat,proto3" json:"ValueSat,omitempty"` N uint32 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` ScriptPubKeyHex []byte `protobuf:"bytes,3,opt,name=ScriptPubKeyHex,proto3" json:"ScriptPubKeyHex,omitempty"` Addresses []string `protobuf:"bytes,4,rep,name=Addresses" json:"Addresses,omitempty"` @@ -159,11 +159,11 @@ func (m *ProtoTransaction_VoutType) String() string { return proto.Co func (*ProtoTransaction_VoutType) ProtoMessage() {} func (*ProtoTransaction_VoutType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 1} } -func (m *ProtoTransaction_VoutType) GetValue() float64 { +func (m *ProtoTransaction_VoutType) GetValueSat() []byte { if m != nil { - return m.Value + return m.ValueSat } - return 0 + return nil } func (m *ProtoTransaction_VoutType) GetN() uint32 { @@ -196,26 +196,26 @@ func init() { func init() { proto.RegisterFile("tx.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 330 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x4e, 0xc2, 0x40, - 0x10, 0x86, 0xb3, 0xb4, 0x14, 0x18, 0x21, 0x92, 0x89, 0x31, 0x0d, 0xf1, 0x50, 0x39, 0xf5, 0xd4, - 0x03, 0xc6, 0x07, 0x50, 0x2f, 0x24, 0x1a, 0x42, 0x16, 0xd2, 0x7b, 0x5b, 0x36, 0xb0, 0x11, 0x77, - 0xb1, 0xdd, 0x1a, 0x78, 0x16, 0x1f, 0xc1, 0x97, 0x34, 0x3b, 0x2d, 0x45, 0x48, 0xbc, 0xed, 0xff, - 0xef, 0x4c, 0xff, 0x6f, 0xff, 0x14, 0xba, 0x66, 0x1f, 0xed, 0x72, 0x6d, 0x34, 0x7a, 0x69, 0xb6, - 0x49, 0xa4, 0x1a, 0x7f, 0xbb, 0x30, 0x9c, 0x5b, 0x67, 0x99, 0x27, 0xaa, 0x48, 0x32, 0x23, 0xb5, - 0x42, 0x04, 0x77, 0xb9, 0x97, 0x2b, 0x9f, 0x05, 0x2c, 0xec, 0x73, 0x3a, 0xe3, 0x10, 0x9c, 0xa9, - 0xd8, 0xfb, 0x2d, 0xb2, 0xec, 0x11, 0xef, 0xa0, 0xf7, 0xbc, 0xd5, 0xd9, 0xbb, 0x91, 0x1f, 0xc2, - 0x77, 0x02, 0x16, 0xba, 0xfc, 0x64, 0xe0, 0x08, 0xba, 0x6f, 0xc7, 0x4b, 0x37, 0x60, 0xe1, 0x80, - 0x37, 0x1a, 0x6f, 0xc1, 0x9b, 0x0a, 0xb9, 0xde, 0x18, 0xbf, 0x4d, 0x37, 0xb5, 0xc2, 0x09, 0x38, - 0xb1, 0x54, 0xbe, 0x17, 0x38, 0xe1, 0xd5, 0x24, 0x88, 0x2a, 0xc4, 0xe8, 0x12, 0x2f, 0x8a, 0xa5, - 0x5a, 0x1e, 0x76, 0x82, 0xdb, 0x61, 0x7c, 0x04, 0x37, 0xd6, 0xa5, 0xf1, 0x3b, 0xb4, 0x74, 0xff, - 0xff, 0x92, 0x2e, 0x0d, 0x6d, 0xd1, 0xf8, 0xe8, 0x87, 0x41, 0xa7, 0xfe, 0x8e, 0x45, 0x7d, 0xd1, - 0x52, 0xa5, 0x49, 0x21, 0xe8, 0xc9, 0x3d, 0xde, 0xe8, 0xa6, 0x8a, 0xd6, 0x9f, 0x2a, 0xb0, 0x8e, - 0x74, 0x08, 0x9e, 0xce, 0x38, 0x86, 0xfe, 0x22, 0xcb, 0xe5, 0xce, 0x2c, 0xe4, 0xda, 0xf6, 0xe4, - 0xd2, 0xfc, 0x99, 0x67, 0x73, 0x16, 0xe2, 0xb3, 0x14, 0x2a, 0x13, 0xf5, 0xc3, 0x1b, 0x6d, 0xcb, - 0x7c, 0x5a, 0xad, 0x72, 0x51, 0x14, 0xa2, 0xa0, 0x02, 0x7a, 0xfc, 0x64, 0x8c, 0xbe, 0xa0, 0x7b, - 0xe4, 0xc7, 0x1b, 0x68, 0xc7, 0xc9, 0xb6, 0xac, 0x50, 0x19, 0xaf, 0x04, 0xf6, 0x81, 0xcd, 0x08, - 0x72, 0xc0, 0xd9, 0x0c, 0x43, 0xb8, 0xae, 0x92, 0xe7, 0x65, 0xfa, 0x2a, 0x0e, 0x16, 0xc8, 0x21, - 0xa0, 0x4b, 0xfb, 0x3c, 0xd7, 0xbd, 0xc8, 0x4d, 0x3d, 0xfa, 0x59, 0x1e, 0x7e, 0x03, 0x00, 0x00, - 0xff, 0xff, 0x64, 0x1d, 0x38, 0xac, 0x38, 0x02, 0x00, 0x00, + // 331 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xc1, 0x6e, 0xea, 0x30, + 0x10, 0x94, 0x89, 0x5f, 0x80, 0x7d, 0xa0, 0x87, 0xf6, 0xf0, 0x14, 0xa1, 0x1e, 0x52, 0x4e, 0x39, + 0xe5, 0x40, 0xd5, 0x0f, 0x68, 0x7b, 0x41, 0x6a, 0x85, 0x90, 0x83, 0x72, 0x4f, 0x82, 0x05, 0x56, + 0xa9, 0x4d, 0x13, 0x47, 0x02, 0xa9, 0x3f, 0xd3, 0x73, 0x7f, 0xb2, 0xf2, 0x12, 0x42, 0x41, 0xea, + 0x6d, 0x67, 0xbc, 0xe3, 0x19, 0x4f, 0x02, 0x3d, 0xbb, 0x8f, 0x77, 0xa5, 0xb1, 0x06, 0xfd, 0xbc, + 0xd8, 0x64, 0x4a, 0x4f, 0x3e, 0x39, 0x8c, 0x16, 0x8e, 0x59, 0x96, 0x99, 0xae, 0xb2, 0xc2, 0x2a, + 0xa3, 0x11, 0x81, 0x2f, 0xf7, 0x6a, 0x15, 0xb0, 0x90, 0x45, 0x03, 0x41, 0x33, 0x8e, 0xc0, 0x9b, + 0xc9, 0x7d, 0xd0, 0x21, 0xca, 0x8d, 0x78, 0x03, 0xfd, 0xc7, 0xad, 0x29, 0x5e, 0xad, 0x7a, 0x93, + 0x81, 0x17, 0xb2, 0x88, 0x8b, 0x33, 0x81, 0x63, 0xe8, 0xbd, 0x9c, 0x0e, 0x79, 0xc8, 0xa2, 0xa1, + 0x68, 0x31, 0xfe, 0x07, 0x7f, 0x26, 0xd5, 0x7a, 0x63, 0x83, 0x3f, 0x74, 0xd2, 0x20, 0x9c, 0x82, + 0x97, 0x2a, 0x1d, 0xf8, 0xa1, 0x17, 0xfd, 0x9d, 0x86, 0xf1, 0x31, 0x62, 0x7c, 0x1d, 0x2f, 0x4e, + 0x95, 0x5e, 0x1e, 0x76, 0x52, 0xb8, 0x65, 0xbc, 0x07, 0x9e, 0x9a, 0xda, 0x06, 0x5d, 0x12, 0xdd, + 0xfe, 0x2e, 0x32, 0xb5, 0x25, 0x15, 0xad, 0x8f, 0xbf, 0x18, 0x74, 0x9b, 0x7b, 0x5c, 0xd4, 0x27, + 0xa3, 0x74, 0x9e, 0x55, 0x92, 0x9e, 0xdc, 0x17, 0x2d, 0x6e, 0xab, 0xe8, 0xfc, 0xa8, 0x02, 0x1b, + 0x4b, 0x8f, 0xc2, 0xd3, 0x8c, 0x13, 0x18, 0x24, 0x45, 0xa9, 0x76, 0x36, 0x51, 0x6b, 0xd7, 0x13, + 0xa7, 0xfd, 0x0b, 0xce, 0xf9, 0x24, 0xf2, 0xbd, 0x96, 0xba, 0x90, 0xcd, 0xc3, 0x5b, 0xec, 0xca, + 0x7c, 0x58, 0xad, 0x4a, 0x59, 0x55, 0xb2, 0xa2, 0x02, 0xfa, 0xe2, 0x4c, 0x8c, 0x3f, 0xa0, 0x77, + 0xca, 0xef, 0x6e, 0x49, 0xb3, 0x6d, 0x2d, 0x93, 0xcc, 0x36, 0x1f, 0xa8, 0xc5, 0x38, 0x00, 0x36, + 0xa7, 0xa8, 0x43, 0xc1, 0xe6, 0x18, 0xc1, 0xbf, 0xa3, 0xff, 0xa2, 0xce, 0x9f, 0xe5, 0xc1, 0xc5, + 0xf2, 0x48, 0x70, 0x4d, 0x5f, 0xba, 0xf3, 0x2b, 0xf7, 0xdc, 0xa7, 0x5f, 0xe6, 0xee, 0x3b, 0x00, + 0x00, 0xff, 0xff, 0x95, 0x9e, 0x00, 0x7a, 0x3e, 0x02, 0x00, 0x00, } diff --git a/bchain/tx.proto b/bchain/tx.proto index 731b8bf5..61e06aed 100644 --- a/bchain/tx.proto +++ b/bchain/tx.proto @@ -11,7 +11,7 @@ syntax = "proto3"; repeated string Addresses = 6; } message VoutType { - double Value = 1; + bytes ValueSat = 1; uint32 N = 2; bytes ScriptPubKeyHex = 3; repeated string Addresses = 4; diff --git a/bchain/types.go b/bchain/types.go index e2aa7276..cfe90771 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math/big" ) // errors with specific meaning returned by blockchain rpc @@ -49,7 +50,8 @@ type Address interface { } type Vout struct { - Value float64 `json:"value"` + ValueSat big.Int + JsonValue json.Number `json:"value"` N uint32 `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` Address Address @@ -89,18 +91,20 @@ type BlockHeader struct { } type MempoolEntry struct { - Size uint32 `json:"size"` - Fee float64 `json:"fee"` - ModifiedFee float64 `json:"modifiedfee"` - Time float64 `json:"time"` - Height uint32 `json:"height"` - DescendantCount uint32 `json:"descendantcount"` - DescendantSize uint32 `json:"descendantsize"` - DescendantFees uint32 `json:"descendantfees"` - AncestorCount uint32 `json:"ancestorcount"` - AncestorSize uint32 `json:"ancestorsize"` - AncestorFees uint32 `json:"ancestorfees"` - Depends []string `json:"depends"` + Size uint32 `json:"size"` + FeeSat big.Int + Fee json.Number `json:"fee"` + ModifiedFeeSat big.Int + ModifiedFee json.Number `json:"modifiedfee"` + Time uint64 `json:"time"` + Height uint32 `json:"height"` + DescendantCount uint32 `json:"descendantcount"` + DescendantSize uint32 `json:"descendantsize"` + DescendantFees uint32 `json:"descendantfees"` + AncestorCount uint32 `json:"ancestorcount"` + AncestorSize uint32 `json:"ancestorsize"` + AncestorFees uint32 `json:"ancestorfees"` + Depends []string `json:"depends"` } type RPCError struct { @@ -132,8 +136,8 @@ type BlockChain interface { GetMempool() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) - EstimateSmartFee(blocks int, conservative bool) (float64, error) - EstimateFee(blocks int) (float64, error) + EstimateSmartFee(blocks int, conservative bool) (big.Int, error) + EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) // mempool ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) @@ -145,7 +149,7 @@ type BlockChain interface { // BlockChainParser defines common interface to parsing and conversions of block chain data type BlockChainParser interface { - // self description + // chain configuration description // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db IsUTXOChain() bool @@ -153,6 +157,11 @@ type BlockChainParser interface { // and used in case of fork // if 0 the blockaddresses column is not used at all (usually non UTXO chains) KeepBlockAddresses() int + // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place + AmountToDecimalString(a *big.Int) string + // AmountToBigInt converts amount in json.Number (string) to big.Int + // it uses string operations to avoid problems with rounding + AmountToBigInt(n json.Number) (big.Int, error) // address id conversions GetAddrIDFromVout(output *Vout) ([]byte, error) GetAddrIDFromAddress(address string) ([]byte, error) diff --git a/blockbook.go b/blockbook.go index cea2c5b6..7e3da52b 100644 --- a/blockbook.go +++ b/blockbook.go @@ -175,7 +175,11 @@ func main() { } index.SetInternalState(internalState) if internalState.DbState != common.DbStateClosed { - glog.Warning("internalState: database in not closed state ", internalState.DbState, ", possibly previous ungraceful shutdown") + if internalState.DbState == common.DbStateInconsistent { + glog.Error("internalState: database is in inconsistent state and cannot be used") + return + } + glog.Warning("internalState: database was left in open state, possibly previous ungraceful shutdown") } if *computeColumnStats { diff --git a/common/internalstate.go b/common/internalstate.go index 8e7b2b61..1053649c 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -11,6 +11,8 @@ const ( DbStateClosed = uint32(iota) // DbStateOpen means db is open or application died without closing the db DbStateOpen + // DbStateInconsistent means db is in inconsistent state and cannot be used + DbStateInconsistent ) // InternalStateColumn contains the data of a db column diff --git a/db/rocksdb.go b/db/rocksdb.go index 30285dab..1141df55 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -6,6 +6,7 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" "os" "path/filepath" "time" @@ -21,7 +22,8 @@ import ( // when doing huge scan, it is better to close it and reopen from time to time to free the resources const refreshIterator = 5000000 const packedHeightBytes = 4 -const dbVersion = 0 +const dbVersion = 3 +const maxAddrIDLen = 1024 // RepairRocksDB calls RocksDb db repair function func RepairRocksDB(name string) error { @@ -46,12 +48,13 @@ const ( cfDefault = iota cfHeight cfAddresses - cfUnspentTxs + cfTxAddresses + cfAddressBalance + cfBlockTxs cfTransactions - cfBlockAddresses ) -var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"} +var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { c := gorocksdb.NewLRUCache(8 << 30) // 8GB @@ -61,37 +64,49 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) bbto.SetBlockCache(c) bbto.SetFilterPolicy(fp) - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) - opts.SetMaxBackgroundCompactions(4) - opts.SetMaxBackgroundFlushes(2) - opts.SetBytesPerSync(1 << 20) // 1MB - opts.SetWriteBufferSize(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) - opts.SetCompression(gorocksdb.NoCompression) + optsNoCompression := gorocksdb.NewDefaultOptions() + optsNoCompression.SetBlockBasedTableFactory(bbto) + optsNoCompression.SetCreateIfMissing(true) + optsNoCompression.SetCreateIfMissingColumnFamilies(true) + optsNoCompression.SetMaxBackgroundCompactions(4) + optsNoCompression.SetMaxBackgroundFlushes(2) + optsNoCompression.SetBytesPerSync(1 << 20) // 1MB + optsNoCompression.SetWriteBufferSize(1 << 27) // 128MB + optsNoCompression.SetMaxOpenFiles(25000) + optsNoCompression.SetCompression(gorocksdb.NoCompression) - // opts for outputs are different: + optsLZ4 := gorocksdb.NewDefaultOptions() + optsLZ4.SetBlockBasedTableFactory(bbto) + optsLZ4.SetCreateIfMissing(true) + optsLZ4.SetCreateIfMissingColumnFamilies(true) + optsLZ4.SetMaxBackgroundCompactions(4) + optsLZ4.SetMaxBackgroundFlushes(2) + optsLZ4.SetBytesPerSync(1 << 20) // 1MB + optsLZ4.SetWriteBufferSize(1 << 27) // 128MB + optsLZ4.SetMaxOpenFiles(25000) + optsLZ4.SetCompression(gorocksdb.LZ4HCCompression) + + // opts for addresses are different: // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter - bbtoOutputs := gorocksdb.NewDefaultBlockBasedTableOptions() - bbtoOutputs.SetBlockSize(16 << 10) // 16kB - bbtoOutputs.SetBlockCache(c) // 8GB + bbtoAddresses := gorocksdb.NewDefaultBlockBasedTableOptions() + bbtoAddresses.SetBlockSize(16 << 10) // 16kB + bbtoAddresses.SetBlockCache(c) // 8GB - optsOutputs := gorocksdb.NewDefaultOptions() - optsOutputs.SetBlockBasedTableFactory(bbtoOutputs) - optsOutputs.SetCreateIfMissing(true) - optsOutputs.SetCreateIfMissingColumnFamilies(true) - optsOutputs.SetMaxBackgroundCompactions(4) - optsOutputs.SetMaxBackgroundFlushes(2) - optsOutputs.SetBytesPerSync(1 << 20) // 1MB - optsOutputs.SetWriteBufferSize(1 << 27) // 128MB - optsOutputs.SetMaxOpenFiles(25000) - optsOutputs.SetCompression(gorocksdb.NoCompression) + optsAddresses := gorocksdb.NewDefaultOptions() + optsAddresses.SetBlockBasedTableFactory(bbtoAddresses) + optsAddresses.SetCreateIfMissing(true) + optsAddresses.SetCreateIfMissingColumnFamilies(true) + optsAddresses.SetMaxBackgroundCompactions(4) + optsAddresses.SetMaxBackgroundFlushes(2) + optsAddresses.SetBytesPerSync(1 << 20) // 1MB + optsAddresses.SetWriteBufferSize(1 << 27) // 128MB + optsAddresses.SetMaxOpenFiles(25000) + optsAddresses.SetCompression(gorocksdb.LZ4HCCompression) - fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts} + // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions + fcOptions := []*gorocksdb.Options{optsLZ4, optsLZ4, optsAddresses, optsLZ4, optsLZ4, optsLZ4, optsLZ4} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) + db, cfh, err := gorocksdb.OpenDbColumnFamilies(optsNoCompression, path, cfNames, fcOptions) if err != nil { return nil, nil, err } @@ -101,7 +116,7 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { - glog.Infof("rocksdb: open %s", path) + glog.Infof("rocksdb: open %s, version %v", path, dbVersion) db, cfh, err := openDB(path) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() @@ -185,11 +200,11 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f for _, o := range outpoints { var vout uint32 var isOutput bool - if o.vout < 0 { - vout = uint32(^o.vout) + if o.index < 0 { + vout = uint32(^o.index) isOutput = false } else { - vout = uint32(o.vout) + vout = uint32(o.index) isOutput = true } tx, err := d.chainParser.UnpackTxid(o.btxID) @@ -238,7 +253,27 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return err } if isUTXO { - if err := d.writeAddressesUTXO(wb, block, op); err != nil { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } + addresses := make(map[string][]outpoint) + txAddressesMap := make(map[string]*txAddresses) + balances := make(map[string]*addrBalance) + if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { + return err + } + if err := d.storeAddresses(wb, block.Height, addresses); err != nil { + return err + } + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { return err } } else { @@ -250,181 +285,338 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { return d.db.Write(d.wo, wb) } +// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type bulkAddresses struct { + height uint32 + addresses map[string][]outpoint +} + +type BulkConnect struct { + d *RocksDB + isUTXO bool + bulkAddresses []bulkAddresses + bulkAddressesCount int + txAddressesMap map[string]*txAddresses + balances map[string]*addrBalance + height uint32 +} + +const ( + maxBulkAddresses = 400000 + maxBulkTxAddresses = 2000000 + partialStoreAddresses = maxBulkTxAddresses / 10 + maxBulkBalances = 2500000 + partialStoreBalances = maxBulkBalances / 10 +) + +func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { + bc := &BulkConnect{ + d: d, + isUTXO: d.chainParser.IsUTXOChain(), + txAddressesMap: make(map[string]*txAddresses), + balances: make(map[string]*addrBalance), + } + if err := d.SetInconsistentState(true); err != nil { + return nil, err + } + glog.Info("rocksdb: bulk connect init, db set to inconsistent state") + return bc, nil +} + +func (b *BulkConnect) storeTxAddresses(c chan error, all bool) { + defer close(c) + start := time.Now() + var txm map[string]*txAddresses + var sp int + if all { + txm = b.txAddressesMap + b.txAddressesMap = make(map[string]*txAddresses) + } else { + txm = make(map[string]*txAddresses) + for k, a := range b.txAddressesMap { + // store all completely spent transactions, they will not be modified again + r := true + for _, o := range a.outputs { + if o.spent == false { + r = false + break + } + } + if r { + txm[k] = a + delete(b.txAddressesMap, k) + } + } + sp = len(txm) + // store some other random transactions if necessary + if len(txm) < partialStoreAddresses { + for k, a := range b.txAddressesMap { + txm[k] = a + delete(b.txAddressesMap, k) + if len(txm) >= partialStoreAddresses { + break + } + } + } + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.storeTxAddresses(wb, txm); err != nil { + c <- err + } else { + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + } + } + glog.Info("rocksdb: height ", b.height, ", stored ", len(txm), " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) +} + +func (b *BulkConnect) storeBalances(c chan error, all bool) { + defer close(c) + start := time.Now() + var bal map[string]*addrBalance + if all { + bal = b.balances + b.balances = make(map[string]*addrBalance) + } else { + bal = make(map[string]*addrBalance) + // store some random balances + for k, a := range b.balances { + bal[k] = a + delete(b.balances, k) + if len(bal) >= partialStoreBalances { + break + } + } + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + if err := b.d.storeBalances(wb, bal); err != nil { + c <- err + } else { + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + } + } + glog.Info("rocksdb: height ", b.height, ", stored ", len(bal), " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) +} + +func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { + for _, ba := range b.bulkAddresses { + if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { + return err + } + } + b.bulkAddressesCount = 0 + b.bulkAddresses = b.bulkAddresses[:0] + return nil +} + +func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { + b.height = block.Height + if !b.isUTXO { + return b.d.ConnectBlock(block) + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + addresses := make(map[string][]outpoint) + if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { + return err + } + start := time.Now() + var sa bool + var storeAddressesChan, storeBalancesChan chan error + if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + sa = true + if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { + storeAddressesChan = make(chan error) + go b.storeTxAddresses(storeAddressesChan, false) + } + if len(b.balances)+partialStoreBalances > maxBulkBalances { + storeBalancesChan = make(chan error) + go b.storeBalances(storeBalancesChan, false) + } + } + b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ + height: block.Height, + addresses: addresses, + }) + b.bulkAddressesCount += len(addresses) + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + } + if storeBlockTxs { + if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { + return err + } + } + if err := b.d.writeHeight(wb, block, opInsert); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + } + if storeAddressesChan != nil { + if err := <-storeAddressesChan; err != nil { + return err + } + } + if storeBalancesChan != nil { + if err := <-storeBalancesChan; err != nil { + return err + } + } + return nil +} + +func (b *BulkConnect) Close() error { + glog.Info("rocksdb: bulk connect closing") + start := time.Now() + storeAddressesChan := make(chan error) + go b.storeTxAddresses(storeAddressesChan, true) + storeBalancesChan := make(chan error) + go b.storeBalances(storeBalancesChan, true) + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + if err := <-storeAddressesChan; err != nil { + return err + } + if err := <-storeBalancesChan; err != nil { + return err + } + if err := b.d.SetInconsistentState(false); err != nil { + return err + } + glog.Info("rocksdb: bulk connect closed, db set to open state") + b.d = nil + return nil +} + // Addresses index type outpoint struct { btxID []byte - vout int32 + index int32 } -func (d *RocksDB) packBlockAddress(addrID []byte, spentTxs map[string][]outpoint) []byte { - vBuf := make([]byte, vlq.MaxLen32) - vl := packVarint(int32(len(addrID)), vBuf) - blockAddress := append([]byte(nil), vBuf[:vl]...) - blockAddress = append(blockAddress, addrID...) - if spentTxs == nil { - } else { - addrUnspentTxs := spentTxs[string(addrID)] - vl = packVarint(int32(len(addrUnspentTxs)), vBuf) - blockAddress = append(blockAddress, vBuf[:vl]...) - buf := d.packOutpoints(addrUnspentTxs) - blockAddress = append(blockAddress, buf...) - } - return blockAddress +type txInput struct { + addrID []byte + valueSat big.Int } -func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, addresses map[string][]outpoint, spentTxs map[string][]outpoint) error { - keep := d.chainParser.KeepBlockAddresses() - blockAddresses := make([]byte, 0) - for addrID, outpoints := range addresses { - baddrID := []byte(addrID) - key := packAddressKey(baddrID, block.Height) - switch op { - case opInsert: - val := d.packOutpoints(outpoints) - wb.PutCF(d.cfh[cfAddresses], key, val) - if keep > 0 { - // collect all addresses be stored in blockaddresses - // they are used in disconnect blocks - blockAddress := d.packBlockAddress(baddrID, spentTxs) - blockAddresses = append(blockAddresses, blockAddress...) - } - case opDelete: - wb.DeleteCF(d.cfh[cfAddresses], key) - } - } - if keep > 0 && op == opInsert { - // write new block address and txs spent in this block - key := packUint(block.Height) - wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses) - // cleanup old block address - if block.Height > uint32(keep) { - for rh := block.Height - uint32(keep); rh < block.Height; rh-- { - key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return err - } - if val.Size() == 0 { - break - } - val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key) - } - } - } - return nil +type txOutput struct { + addrID []byte + spent bool + valueSat big.Int } -func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { - if len(addrID) > 0 { - if len(addrID) > 1024 { - glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) - } else { - strAddrID := string(addrID) - records[strAddrID] = append(records[strAddrID], outpoint{ - btxID: btxid, - vout: vout, - }) - if op == opDelete { - // remove transactions from cache - d.internalDeleteTx(wb, btxid) - } - } - } - return nil +type txAddresses struct { + inputs []txInput + outputs []txOutput } -func (d *RocksDB) getUnspentTx(btxID []byte) ([]byte, error) { - // find it in db, in the column cfUnspentTxs - val, err := d.db.GetCF(d.ro, d.cfh[cfUnspentTxs], btxID) +type addrBalance struct { + txs uint32 + sentSat big.Int + balanceSat big.Int +} + +type blockTxs struct { + btxID []byte + inputs []outpoint +} + +func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrID []byte, logText string) { + ad, err := d.chainParser.OutputScriptToAddresses(addrID) + had := hex.EncodeToString(addrID) if err != nil { - return nil, err + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", had, logText, valueSat.String(), err) + } else { + glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, had, logText, valueSat.String()) } - defer val.Free() - data := append([]byte(nil), val.Data()...) - return data, nil + valueSat.SetInt64(0) } -func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) []byte { - // resize the addr buffer if necessary by a new estimate - if cap(txAddrs)-len(txAddrs) < 2*vlq.MaxLen32+len(addrID) { - txAddrs = append(txAddrs, make([]byte, vlq.MaxLen32+len(addrID)+remaining*32)...)[:len(txAddrs)] - } - // addrID is packed as number of bytes of the addrID + bytes of addrID + vout - lv := packVarint(int32(len(addrID)), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - txAddrs = append(txAddrs, addrID...) - lv = packVarint(int32(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - return txAddrs -} - -func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, []byte) { - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(unspentAddrs); { - l, lv1 := unpackVarint(unspentAddrs[i:]) - // index of vout of address in unspentAddrs - j := i + int(l) + lv1 - if j >= len(unspentAddrs) { - glog.Error("rocksdb: Inconsistent data in unspentAddrs ", hex.EncodeToString(unspentAddrs), ", ", vout) - return nil, unspentAddrs - } - n, lv2 := unpackVarint(unspentAddrs[j:]) - if uint32(n) == vout { - addrID := append([]byte(nil), unspentAddrs[i+lv1:j]...) - unspentAddrs = append(unspentAddrs[:i], unspentAddrs[j+lv2:]...) - return addrID, unspentAddrs - } - i = j + lv2 - } - return nil, unspentAddrs -} - -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } - addresses := make(map[string][]outpoint) - unspentTxs := make(map[string][]byte) - thisBlockTxs := make(map[string]struct{}) - btxIDs := make([][]byte, len(block.Txs)) - // first process all outputs, build mapping of addresses to outpoints and mappings of unspent txs to addresses - for txi, tx := range block.Txs { +func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*txAddresses, balances map[string]*addrBalance) error { + blockTxIDs := make([][]byte, len(block.Txs)) + // first process all outputs so that inputs can point to txs in this block + for txi := range block.Txs { + tx := &block.Txs[txi] btxID, err := d.chainParser.PackTxid(tx.Txid) if err != nil { return err } - btxIDs[txi] = btxID - // preallocate estimated size of addresses (32 bytes is 1 byte length of addrID, 25 bytes addrID, 1-2 bytes vout and reserve) - txAddrs := make([]byte, 0, len(tx.Vout)*32) + blockTxIDs[txi] = btxID + ta := txAddresses{} + ta.outputs = make([]txOutput, len(tx.Vout)) + txAddressesMap[string(btxID)] = &ta for i, output := range tx.Vout { + tao := &ta.outputs[i] + tao.valueSat = output.ValueSat addrID, err := d.chainParser.GetAddrIDFromVout(&output) - if err != nil { - // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) - if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + if err != nil || len(addrID) == 0 || len(addrID) > maxAddrIDLen { + if err != nil { + // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + } + } else { + glog.Infof("rocksdb: height %d, tx %v, vout %v, skipping addrID of length %d", block.Height, tx.Txid, i, len(addrID)) } continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(output.N), block.Height) - if err != nil { - return err + tao.addrID = addrID + strAddrID := string(addrID) + // check that the address was used already in this block + o, processed := addresses[strAddrID] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, btxID) } - txAddrs = appendPackedAddrID(txAddrs, addrID, output.N, len(tx.Vout)-i) + addresses[strAddrID] = append(o, outpoint{ + btxID: btxID, + index: int32(i), + }) + ab, e := balances[strAddrID] + if !e { + ab, err = d.getAddressBalance(addrID) + if err != nil { + return err + } + if ab == nil { + ab = &addrBalance{} + } + balances[strAddrID] = ab + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.txs++ + } + ab.balanceSat.Add(&ab.balanceSat, &output.ValueSat) } - stxID := string(btxID) - unspentTxs[stxID] = txAddrs - thisBlockTxs[stxID] = struct{}{} } - // locate addresses spent by this tx and remove them from unspent addresses - // keep them so that they be stored for DisconnectBlock functionality - spentTxs := make(map[string][]outpoint) - for txi, tx := range block.Txs { - spendingTxid := btxIDs[txi] + // process inputs + for txi := range block.Txs { + tx := &block.Txs[txi] + spendingTxid := blockTxIDs[txi] + ta := txAddressesMap[string(spendingTxid)] + ta.inputs = make([]txInput, len(tx.Vin)) for i, input := range tx.Vin { + tai := &ta.inputs[i] btxID, err := d.chainParser.PackTxid(input.Txid) if err != nil { // do not process inputs without input txid @@ -433,50 +625,374 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } return err } - // find the tx in current block or already processed stxID := string(btxID) - unspentAddrs, exists := unspentTxs[stxID] - if !exists { - // else find it in previous blocks - unspentAddrs, err = d.getUnspentTx(btxID) + ita, e := txAddressesMap[stxID] + if !e { + ita, err = d.getTxAddresses(btxID) if err != nil { return err } - if unspentAddrs == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v missing in unspentTxs", block.Height, tx.Txid, input.Txid, input.Vout, i) + if ita == nil { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v not found in txAddresses", block.Height, tx.Txid, input.Txid) continue } + txAddressesMap[stxID] = ita } - var addrID []byte - addrID, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout) - if addrID == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v not found in unspentAddrs", block.Height, tx.Txid, input.Txid, input.Vout, i) + if len(ita.outputs) <= int(input.Vout) { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) continue } - // record what was spent in this tx - // skip transactions that were created in this block - if _, exists := thisBlockTxs[stxID]; !exists { - saddrID := string(addrID) - rut := spentTxs[saddrID] - rut = append(rut, outpoint{btxID, int32(input.Vout)}) - spentTxs[saddrID] = rut + ot := &ita.outputs[int(input.Vout)] + if ot.spent { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, spendingTxid, int32(^i), block.Height) + tai.addrID = ot.addrID + tai.valueSat = ot.valueSat + // mark the output as spent in tx + ot.spent = true + if len(ot.addrID) == 0 { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) + continue + } + strAddrID := string(ot.addrID) + // check that the address was used already in this block + o, processed := addresses[strAddrID] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, spendingTxid) + } + addresses[strAddrID] = append(o, outpoint{ + btxID: spendingTxid, + index: ^int32(i), + }) + ab, e := balances[strAddrID] + if !e { + ab, err = d.getAddressBalance(ot.addrID) + if err != nil { + return err + } + if ab == nil { + ab = &addrBalance{} + } + balances[strAddrID] = ab + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.txs++ + } + ab.balanceSat.Sub(&ab.balanceSat, &ot.valueSat) + if ab.balanceSat.Sign() < 0 { + d.resetValueSatToZero(&ab.balanceSat, ot.addrID, "balance") + } + ab.sentSat.Add(&ab.sentSat, &ot.valueSat) + } + } + return nil +} + +func processedInTx(o []outpoint, btxID []byte) bool { + for _, op := range o { + if bytes.Equal(btxID, op.btxID) { + return true + } + } + return false +} + +func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error { + for addrID, outpoints := range addresses { + ba := []byte(addrID) + key := packAddressKey(ba, height) + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + } + return nil +} + +func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*txAddresses) error { + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) + for txID, ta := range am { + buf = packTxAddresses(ta, buf, varBuf) + wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) + } + return nil +} + +func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*addrBalance) error { + // allocate buffer big enough for number of txs + 2 bigints + buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) + for addrID, ab := range abm { + // balance with 0 transactions is removed from db - happens in disconnect + if ab == nil || ab.txs <= 0 { + wb.DeleteCF(d.cfh[cfAddressBalance], []byte(addrID)) + } else { + l := packVaruint(uint(ab.txs), buf) + ll := packBigint(&ab.sentSat, buf[l:]) + l += ll + ll = packBigint(&ab.balanceSat, buf[l:]) + l += ll + wb.PutCF(d.cfh[cfAddressBalance], []byte(addrID), buf[:l]) + } + } + return nil +} + +func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, 0, pl*len(block.Txs)) + varBuf := make([]byte, vlq.MaxLen64) + zeroTx := make([]byte, pl) + for i := range block.Txs { + tx := &block.Txs[i] + o := make([]outpoint, len(tx.Vin)) + for v := range tx.Vin { + vin := &tx.Vin[v] + btxID, err := d.chainParser.PackTxid(vin.Txid) + if err != nil { + // do not process inputs without input txid + if err == bchain.ErrTxidMissing { + btxID = zeroTx + } else { + return err + } + } + o[v].btxID = btxID + o[v].index = int32(vin.Vout) + } + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return err + } + buf = append(buf, btxID...) + l := packVaruint(uint(len(o)), varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, d.packOutpoints(o)...) + } + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockTxs], key, buf) + keep := d.chainParser.KeepBlockAddresses() + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh < block.Height; rh-- { + key = packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) if err != nil { return err } - unspentTxs[stxID] = unspentAddrs + if val.Size() == 0 { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) } } - if err := d.writeAddressRecords(wb, block, op, addresses, spentTxs); err != nil { - return err + return nil +} + +func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { + pl := d.chainParser.PackedTxidLen() + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) + if err != nil { + return nil, err } - // save unspent txs from current block - for tx, val := range unspentTxs { - if len(val) == 0 { - wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) + defer val.Free() + buf := val.Data() + bt := make([]blockTxs, 0) + for i := 0; i < len(buf); { + if len(buf)-i < pl { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + txid := make([]byte, pl) + copy(txid, buf[i:]) + i += pl + o, ol, err := d.unpackNOutpoints(buf[i:]) + if err != nil { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + bt = append(bt, blockTxs{ + btxID: txid, + inputs: o, + }) + i += ol + } + return bt, nil +} + +func (d *RocksDB) getAddressBalance(addrID []byte) (*addrBalance, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 3 is minimum length of addrBalance - 1 byte txs, 1 byte sent, 1 byte balance + if len(buf) < 3 { + return nil, nil + } + txs, l := unpackVaruint(buf) + sentSat, sl := unpackBigint(buf[l:]) + balanceSat, _ := unpackBigint(buf[l+sl:]) + return &addrBalance{ + txs: uint32(txs), + sentSat: sentSat, + balanceSat: balanceSat, + }, nil +} + +func (d *RocksDB) getTxAddresses(btxID []byte) (*txAddresses, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 2 is minimum length of addrBalance - 1 byte inputs len, 1 byte outputs len + if len(buf) < 2 { + return nil, nil + } + return unpackTxAddresses(buf) +} + +func packTxAddresses(ta *txAddresses, buf []byte, varBuf []byte) []byte { + buf = buf[:0] + l := packVaruint(uint(len(ta.inputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.inputs { + buf = appendTxInput(&ta.inputs[i], buf, varBuf) + } + l = packVaruint(uint(len(ta.outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.outputs { + buf = appendTxOutput(&ta.outputs[i], buf, varBuf) + } + return buf +} + +func appendTxInput(txi *txInput, buf []byte, varBuf []byte) []byte { + la := len(txi.addrID) + l := packVaruint(uint(la), varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txi.addrID...) + l = packBigint(&txi.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func appendTxOutput(txo *txOutput, buf []byte, varBuf []byte) []byte { + la := len(txo.addrID) + if txo.spent { + la = ^la + } + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txo.addrID...) + l = packBigint(&txo.valueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func unpackTxAddresses(buf []byte) (*txAddresses, error) { + ta := txAddresses{} + inputs, l := unpackVaruint(buf) + ta.inputs = make([]txInput, inputs) + for i := uint(0); i < inputs; i++ { + l += unpackTxInput(&ta.inputs[i], buf[l:]) + } + outputs, ll := unpackVaruint(buf[l:]) + l += ll + ta.outputs = make([]txOutput, outputs) + for i := uint(0); i < outputs; i++ { + l += unpackTxOutput(&ta.outputs[i], buf[l:]) + } + return &ta, nil +} + +func unpackTxInput(ti *txInput, buf []byte) int { + al, l := unpackVaruint(buf) + ti.addrID = make([]byte, al) + copy(ti.addrID, buf[l:l+int(al)]) + al += uint(l) + ti.valueSat, l = unpackBigint(buf[al:]) + return l + int(al) +} + +func unpackTxOutput(to *txOutput, buf []byte) int { + al, l := unpackVarint(buf) + if al < 0 { + to.spent = true + al = ^al + } + to.addrID = make([]byte, al) + copy(to.addrID, buf[l:l+al]) + al += l + to.valueSat, l = unpackBigint(buf[al:]) + return l + al +} + +func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { + buf := make([]byte, 0) + bvout := make([]byte, vlq.MaxLen32) + for _, o := range outpoints { + l := packVarint32(o.index, bvout) + buf = append(buf, []byte(o.btxID)...) + buf = append(buf, bvout[:l]...) + } + return buf +} + +func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + outpoints := make([]outpoint, 0) + for i := 0; i < len(buf); { + btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) + i += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[i:]) + i += voutLen + outpoints = append(outpoints, outpoint{ + btxID: btxID, + index: vout, + }) + } + return outpoints, nil +} + +func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + n, p := unpackVaruint(buf) + outpoints := make([]outpoint, n) + for i := uint(0); i < n; i++ { + if p+txidUnpackedLen >= len(buf) { + return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") + } + btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) + p += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[p:]) + p += voutLen + outpoints[i] = outpoint{ + btxID: btxID, + index: vout, + } + } + return outpoints, p, nil +} + +func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { + if len(addrID) > 0 { + if len(addrID) > maxAddrIDLen { + glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) } else { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) + strAddrID := string(addrID) + records[strAddrID] = append(records[strAddrID], outpoint{ + btxID: btxid, + index: vout, + }) + if op == opDelete { + // remove transactions from cache + d.internalDeleteTx(wb, btxid) + } } } return nil @@ -518,98 +1034,17 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. } } } - return d.writeAddressRecords(wb, block, op, addresses, nil) -} - -func (d *RocksDB) unpackBlockAddresses(buf []byte) ([][]byte, [][]outpoint, error) { - addresses := make([][]byte, 0) - outpointsArray := make([][]outpoint, 0) - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(buf); { - l, lv := unpackVarint(buf[i:]) - j := i + int(l) + lv - if j > len(buf) { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addrID := append([]byte(nil), buf[i+lv:j]...) - outpoints, ol, err := d.unpackNOutpoints(buf[j:]) - if err != nil { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addresses = append(addresses, addrID) - outpointsArray = append(outpointsArray, outpoints) - i = j + ol - } - return addresses, outpointsArray, nil -} - -func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { - buf := make([]byte, 0) - bvout := make([]byte, vlq.MaxLen32) - for _, o := range outpoints { - l := packVarint(o.vout, bvout) - buf = append(buf, []byte(o.btxID)...) - buf = append(buf, bvout[:l]...) - } - return buf -} - -func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - outpoints := make([]outpoint, 0) - for i := 0; i < len(buf); { - btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) - i += txidUnpackedLen - vout, voutLen := unpackVarint(buf[i:]) - i += voutLen - outpoints = append(outpoints, outpoint{ - btxID: btxID, - vout: vout, - }) - } - return outpoints, nil -} - -func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - n, p := unpackVarint(buf) - outpoints := make([]outpoint, n) - for i := int32(0); i < n; i++ { - if p+txidUnpackedLen >= len(buf) { - return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") - } - btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) - p += txidUnpackedLen - vout, voutLen := unpackVarint(buf[p:]) - p += voutLen - outpoints[i] = outpoint{ - btxID: btxID, - vout: vout, + for addrID, outpoints := range addresses { + key := packAddressKey([]byte(addrID), block.Height) + switch op { + case opInsert: + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + case opDelete: + wb.DeleteCF(d.cfh[cfAddresses], key) } } - return outpoints, p, nil -} - -func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { - btxid, err := d.chainParser.PackTxid(txid) - if err != nil { - return nil, err - } - bv := make([]byte, vlq.MaxLen32) - l := packVarint(vout, bv) - buf := make([]byte, 0, l+len(btxid)) - buf = append(buf, btxid...) - buf = append(buf, bv[:l]...) - return buf, nil -} - -func (d *RocksDB) unpackOutpoint(buf []byte) (string, int32, int) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - txid, _ := d.chainParser.UnpackTxid(buf[:txidUnpackedLen]) - vout, o := unpackVarint(buf[txidUnpackedLen:]) - return txid, vout, txidUnpackedLen + o + return nil } // Block index @@ -640,11 +1075,7 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { return d.chainParser.UnpackBlockHash(val.Data()) } -func (d *RocksDB) writeHeight( - wb *gorocksdb.WriteBatch, - block *bchain.Block, - op int, -) error { +func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { key := packUint(block.Height) switch op { @@ -657,22 +1088,10 @@ func (d *RocksDB) writeHeight( case opDelete: wb.DeleteCF(d.cfh[cfHeight], key) } - return nil } -func (d *RocksDB) getBlockAddresses(key []byte) ([][]byte, [][]outpoint, error) { - b, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return nil, nil, err - } - defer b.Free() - // block is missing in DB - if b.Data() == nil { - return nil, nil, errors.New("Block addresses missing") - } - return d.unpackBlockAddresses(b.Data()) -} +// Disconnect blocks func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]byte, error) { glog.Infof("db: doing full scan of addresses column") @@ -719,93 +1138,178 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -// DisconnectBlockRange removes all data belonging to blocks in range lower-higher -// it finds the data in blockaddresses column if available, -// otherwise by doing quite slow full scan of addresses column -func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys := [][]byte{} - addrOutpoints := [][]byte{} - addrUnspentOutpoints := [][]outpoint{} - keep := d.chainParser.KeepBlockAddresses() - var err error - if keep > 0 { - for height := lower; height <= higher; height++ { - addresses, unspentOutpoints, err := d.getBlockAddresses(packUint(height)) +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *txAddresses, + txAddressesToUpdate map[string]*txAddresses, balances map[string]*addrBalance) error { + addresses := make(map[string]struct{}) + getAddressBalance := func(addrID []byte) (*addrBalance, error) { + var err error + s := string(addrID) + b, fb := balances[s] + if !fb { + b, err = d.getAddressBalance(addrID) + if err != nil { + return nil, err + } + balances[s] = b + } + return b, nil + } + for i, t := range txa.inputs { + if len(t.addrID) > 0 { + s := string(t.addrID) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.addrID) if err != nil { - glog.Error(err) return err } - for i, addrID := range addresses { - addrKey := packAddressKey(addrID, height) - val, err := d.db.GetCF(d.ro, d.cfh[cfAddresses], addrKey) + if b != nil { + // subtract number of txs only once + if !exist { + b.txs-- + } + b.sentSat.Sub(&b.sentSat, &t.valueSat) + if b.sentSat.Sign() < 0 { + d.resetValueSatToZero(&b.sentSat, t.addrID, "sent amount") + } + b.balanceSat.Add(&b.balanceSat, &t.valueSat) + } else { + ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) + had := hex.EncodeToString(t.addrID) + glog.Warningf("Balance for address %s (%s) not found", ad, had) + } + s = string(inputs[i].btxID) + sa, exist := txAddressesToUpdate[s] + if !exist { + sa, err = d.getTxAddresses(inputs[i].btxID) if err != nil { - glog.Error(err) return err } - addrKeys = append(addrKeys, addrKey) - av := append([]byte(nil), val.Data()...) - val.Free() - addrOutpoints = append(addrOutpoints, av) - addrUnspentOutpoints = append(addrUnspentOutpoints, unspentOutpoints[i]) + txAddressesToUpdate[s] = sa + } + sa.outputs[inputs[i].index].spent = false + } + } + for _, t := range txa.outputs { + if len(t.addrID) > 0 { + s := string(t.addrID) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.addrID) + if err != nil { + return err + } + if b != nil { + // subtract number of txs only once + if !exist { + b.txs-- + } + b.balanceSat.Sub(&b.balanceSat, &t.valueSat) + if b.balanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.balanceSat, t.addrID, "balance") + } + } else { + ad, _ := d.chainParser.OutputScriptToAddresses(t.addrID) + had := hex.EncodeToString(t.addrID) + glog.Warningf("Balance for address %s (%s) not found", ad, had) } } - } else { - addrKeys, addrOutpoints, err = d.allAddressesScan(lower, higher) + } + for a := range addresses { + key := packAddressKey([]byte(a), height) + wb.DeleteCF(d.cfh[cfAddresses], key) + } + return nil +} + +// DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher +// if they are in the range kept in the cfBlockTxids column +func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + blocks := make([][]blockTxs, higher-lower+1) + for height := lower; height <= higher; height++ { + blockTxs, err := d.getBlockTxs(height) if err != nil { return err } + if len(blockTxs) == 0 { + return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) + } + blocks[height-lower] = blockTxs } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + txAddressesToUpdate := make(map[string]*txAddresses) + txsToDelete := make(map[string]struct{}) + balances := make(map[string]*addrBalance) + for height := higher; height >= lower; height-- { + blockTxs := blocks[height-lower] + glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") + // go backwards to avoid interim negative balance + // when connecting block, amount is first in tx on the output side, then in another tx on the input side + // when disconnecting, it must be done backwards + for i := len(blockTxs) - 1; i >= 0; i-- { + txid := blockTxs[i].btxID + s := string(txid) + txsToDelete[s] = struct{}{} + txa, err := d.getTxAddresses(txid) + if err != nil { + return err + } + if txa == nil { + ut, _ := d.chainParser.UnpackTxid(txid) + glog.Warning("TxAddress for txid ", ut, " not found") + continue + } + if err := d.disconnectTxAddresses(wb, height, s, blockTxs[i].inputs, txa, txAddressesToUpdate, balances); err != nil { + return err + } + } + key := packUint(height) + wb.DeleteCF(d.cfh[cfBlockTxs], key) + wb.DeleteCF(d.cfh[cfHeight], key) + } + d.storeTxAddresses(wb, txAddressesToUpdate) + d.storeBalances(wb, balances) + for s := range txsToDelete { + b := []byte(s) + wb.DeleteCF(d.cfh[cfTransactions], b) + wb.DeleteCF(d.cfh[cfTxAddresses], b) + } + err := d.db.Write(d.wo, wb) + if err == nil { + glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) + } + return err +} +// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks +// it is very slow operation +func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + addrKeys, _, err := d.allAddressesScan(lower, higher) + if err != nil { + return err + } glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - unspentTxs := make(map[string][]byte) - for addrIndex, addrKey := range addrKeys { + for _, addrKey := range addrKeys { if glog.V(2) { glog.Info("address ", hex.EncodeToString(addrKey)) } // delete address:height from the index wb.DeleteCF(d.cfh[cfAddresses], addrKey) - addrID, _, err := unpackAddressKey(addrKey) - if err != nil { - return err - } - // recreate unspentTxs, which were spent by this block (that is being disconnected) - for _, o := range addrUnspentOutpoints[addrIndex] { - stxID := string(o.btxID) - txAddrs, exists := unspentTxs[stxID] - if !exists { - txAddrs, err = d.getUnspentTx(o.btxID) - if err != nil { - return err - } - } - txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.vout), 1) - unspentTxs[stxID] = txAddrs - } - // delete unspentTxs from this block - outpoints, err := d.unpackOutpoints(addrOutpoints[addrIndex]) - if err != nil { - return err - } - for _, o := range outpoints { - wb.DeleteCF(d.cfh[cfUnspentTxs], o.btxID) - d.internalDeleteTx(wb, o.btxID) - } - } - for key, val := range unspentTxs { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(key), val) } for height := lower; height <= higher; height++ { if glog.V(2) { glog.Info("height ", height) } - key := packUint(height) - if keep > 0 { - wb.DeleteCF(d.cfh[cfBlockAddresses], key) - } - wb.DeleteCF(d.cfh[cfHeight], key) + wb.DeleteCF(d.cfh[cfHeight], packUint(height)) } err = d.db.Write(d.wo, wb) if err == nil { @@ -948,6 +1452,18 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro return is, nil } +func (d *RocksDB) SetInconsistentState(inconsistent bool) error { + if d.is == nil { + return errors.New("Internal state not created") + } + if inconsistent { + d.is.DbState = common.DbStateInconsistent + } else { + d.is.DbState = common.DbStateOpen + } + return d.storeState(d.is) +} + // SetInternalState sets the InternalState to be used by db to collect internal state func (d *RocksDB) SetInternalState(is *common.InternalState) { d.is = is @@ -955,11 +1471,17 @@ func (d *RocksDB) SetInternalState(is *common.InternalState) { // StoreInternalState stores the internal state to db func (d *RocksDB) StoreInternalState(is *common.InternalState) error { - for c := 0; c < len(cfNames); c++ { - rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) - d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) - d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + if d.metrics != nil { + for c := 0; c < len(cfNames); c++ { + rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) + d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) + d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + } } + return d.storeState(is) +} + +func (d *RocksDB) storeState(is *common.InternalState) error { buf, err := is.Pack() if err != nil { return err @@ -1047,11 +1569,89 @@ func unpackUint(buf []byte) uint32 { return binary.BigEndian.Uint32(buf) } -func packVarint(i int32, buf []byte) int { +func packVarint32(i int32, buf []byte) int { return vlq.PutInt(buf, int64(i)) } -func unpackVarint(buf []byte) (int32, int) { +func packVarint(i int, buf []byte) int { + return vlq.PutInt(buf, int64(i)) +} + +func packVaruint(i uint, buf []byte) int { + return vlq.PutUint(buf, uint64(i)) +} + +func unpackVarint32(buf []byte) (int32, int) { i, ofs := vlq.Int(buf) return int32(i), ofs } + +func unpackVarint(buf []byte) (int, int) { + i, ofs := vlq.Int(buf) + return int(i), ofs +} + +func unpackVaruint(buf []byte) (uint, int) { + i, ofs := vlq.Uint(buf) + return uint(i), ofs +} + +const ( + // number of bits in a big.Word + wordBits = 32 << (uint64(^big.Word(0)) >> 63) + // number of bytes in a big.Word + wordBytes = wordBits / 8 + // max packed bigint words + maxPackedBigintWords = (256 - wordBytes) / wordBytes + maxPackedBigintBytes = 249 +) + +// big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int +// number of written bytes is returned +// limitation: bigints longer than 248 bytes are truncated to 248 bytes +// caution: buffer must be big enough to hold the packed big int, buffer 249 bytes big is always safe +func packBigint(bi *big.Int, buf []byte) int { + w := bi.Bits() + lw := len(w) + // zero returns only one byte - zero length + if lw == 0 { + buf[0] = 0 + return 1 + } + // pack the most significant word in a special way - skip leading zeros + w0 := w[lw-1] + fb := 8 + mask := big.Word(0xff) << (wordBits - 8) + for w0&mask == 0 { + fb-- + mask >>= 8 + } + for i := fb; i > 0; i-- { + buf[i] = byte(w0) + w0 >>= 8 + } + // if the big int is too big (> 2^1984), the number of bytes would not fit to 1 byte + // in this case, truncate the number, it is not expected to work with this big numbers as amounts + s := 0 + if lw > maxPackedBigintWords { + s = lw - maxPackedBigintWords + } + // pack the rest of the words in reverse order + for j := lw - 2; j >= s; j-- { + d := w[j] + for i := fb + wordBytes; i > fb; i-- { + buf[i] = byte(d) + d >>= 8 + } + fb += wordBytes + } + buf[0] = byte(fb) + return fb + 1 +} + +func unpackBigint(buf []byte) (big.Int, int) { + var r big.Int + l := int(buf[0]) + 1 + r.SetBytes(buf[1:l]) + return r, l +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 52be1c5d..adf9e6c9 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -1,17 +1,18 @@ -// +build unittest +// build unittest package db import ( "blockbook/bchain" "blockbook/bchain/coins/btc" + "blockbook/common" "encoding/hex" "fmt" "io/ioutil" + "math/big" "os" "reflect" "sort" - "strconv" "strings" "testing" @@ -22,6 +23,12 @@ import ( // for number n, the packing is: 2*n if n>=0 else 2*(-n)-1 // takes only 1 byte if abs(n)<127 +func bitcoinTestnetParser() *btc.BitcoinParser { + return btc.NewBitcoinParser( + btc.GetChainParams("test"), + &btc.Configuration{BlockAddressesToKeep: 1}) +} + func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { tmp, err := ioutil.TempDir("", "testdb") if err != nil { @@ -47,6 +54,9 @@ func closeAndDestroyRocksDB(t *testing.T, d *RocksDB) { } func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { + if addr == "" { + return "" + } b, err := d.chainParser.AddressToOutputScript(addr) if err != nil { t.Fatal(err) @@ -54,9 +64,25 @@ func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { return hex.EncodeToString(b) } +func inputAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return hex.EncodeToString([]byte{byte(len(h) / 2)}) + h +} + func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)), 16) + h + return hex.EncodeToString([]byte{byte(len(h))}) + h +} + +func spentAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return hex.EncodeToString([]byte{byte(len(h) + 1)}) + h +} + +func bigintToHex(i *big.Int) string { + b := make([]byte, maxPackedBigintBytes) + l := packBigint(i, b) + return hex.EncodeToString(b[:l]) } // keyPair is used to compare given key value in DB with expected @@ -104,7 +130,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { valOK = kp[i].CompareFunc(val) } if !valOK { - return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, cfNames[col], i, kp[i].Value) + return errors.Errorf("Incorrect value %v found in column %v row %v key %v, expecting %v", val, cfNames[col], i, key, kp[i].Value) } i++ } @@ -114,6 +140,41 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } +const ( + txidB1T1 = "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + txidB1T2 = "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + txidB2T1 = "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + txidB2T2 = "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + txidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + txidB2T4 = "fdd824a780cbb718eeb766eb05d83fdefc793a27082cd5e67f856d69798cf7db" + + addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac + addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac + addr3 = "mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw" // 76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac + addr4 = "2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS" // a9144a21db08fb6882cb152e1ff06780a430740f770487 + addr5 = "2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1" // a914e921fc4912a315078f370d959f2c4f7b6d2a683c87 + addr6 = "mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX" // 76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac + addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac + addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac + addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac + addrA = "mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj" // 76a914d03c0d863d189b23b061a95ad32940b65837609f88ac +) + +var ( + satZero = big.NewInt(0) + satB1T1A1 = big.NewInt(100000000) + satB1T1A2 = big.NewInt(12345) + satB1T2A3 = big.NewInt(1234567890123) + satB1T2A4 = big.NewInt(1) + satB1T2A5 = big.NewInt(9876) + satB2T1A6 = big.NewInt(317283951061) + satB2T1A7 = big.NewInt(917283951061) + satB2T2A8 = big.NewInt(118641975500) + satB2T2A9 = big.NewInt(198641975500) + satB2T3A5 = big.NewInt(9000) + satB2T4AA = big.NewInt(1360030331) +) + func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ @@ -122,44 +183,49 @@ func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d), + Hex: addressToPubKeyHex(addr1, t, d), }, + ValueSat: *satB1T1A1, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d), + Hex: addressToPubKeyHex(addr2, t, d), }, + ValueSat: *satB1T1A2, }, }, Blocktime: 22549300000, Time: 22549300000, }, bchain.Tx{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d), + Hex: addressToPubKeyHex(addr3, t, d), }, + ValueSat: *satB1T2A3, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d), + Hex: addressToPubKeyHex(addr4, t, d), }, + ValueSat: *satB1T2A4, }, bchain.Vout{ N: 2, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB1T2A5, }, }, Blocktime: 22549300001, @@ -177,14 +243,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vin: []bchain.Vin{ + // addr3 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 0, }, + // addr2 bchain.Vin{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: 1, }, }, @@ -192,30 +260,32 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d), + Hex: addressToPubKeyHex(addr6, t, d), }, + ValueSat: *satB2T1A6, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d), + Hex: addressToPubKeyHex(addr7, t, d), }, + ValueSat: *satB2T1A7, }, }, Blocktime: 22549400000, Time: 22549400000, }, bchain.Tx{ - Txid: "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", + Txid: txidB2T2, Vin: []bchain.Vin{ - // spending an output in the same block + // spending an output in the same block - addr6 bchain.Vin{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vout: 0, }, - // spending an output in the previous block + // spending an output in the previous block - addr4 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 1, }, }, @@ -223,14 +293,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d), + Hex: addressToPubKeyHex(addr8, t, d), }, + ValueSat: *satB2T2A8, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d), + Hex: addressToPubKeyHex(addr9, t, d), }, + ValueSat: *satB2T2A9, }, }, Blocktime: 22549400001, @@ -238,10 +310,11 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, // transaction from the same address in the previous block bchain.Tx{ - Txid: "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", + Txid: txidB2T3, Vin: []bchain.Vin{ + // addr5 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 2, }, }, @@ -249,18 +322,44 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB2T3A5, }, }, Blocktime: 22549400002, Time: 22549400002, }, + // mining transaction + bchain.Tx{ + Txid: txidB2T4, + Vin: []bchain.Vin{ + bchain.Vin{ + Coinbase: "03bf1e1504aede765b726567696f6e312f50726f6a65637420425443506f6f6c2f01000001bf7e000000000000", + }, + }, + Vout: []bchain.Vout{ + bchain.Vout{ + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex(addrA, t, d), + }, + ValueSat: *satB2T4AA, + }, + bchain.Vout{ + N: 1, + ScriptPubKey: bchain.ScriptPubKey{}, + ValueSat: *satZero, + }, + }, + Blocktime: 22549400003, + Time: 22549400003, + }, }, } } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, }); err != nil { @@ -270,62 +369,63 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { } // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", - }) - }, + txidB1T1, + "00" + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), + nil, }, keyPair{ - "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "04", - }) - }, + txidB1T2, + "00" + "03" + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), + nil, }, }); err != nil { { t.Fatal(err) } } - // after disconnect there are no blockaddresses for the previous block - var blockAddressesKp []keyPair - if noBlockAddresses { - blockAddressesKp = []keyPair{} + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A2), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A3), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A4), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A5), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + var blockTxsKp []keyPair + if afterDisconnect { + blockTxsKp = []keyPair{} } else { - // the values in cfBlockAddresses are in random order, must use CompareFunc - blockAddressesKp = []keyPair{ - keyPair{"000370d5", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "00", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", - }) - }, + blockTxsKp = []keyPair{ + keyPair{ + "000370d5", + txidB1T1 + "00" + txidB1T2 + "00", + nil, }, } } - if err := checkColumn(d, cfBlockAddresses, blockAddressesKp); err != nil { + + if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil { { t.Fatal(err) } @@ -342,47 +442,75 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { } } if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, - keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil}, - keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil}, - keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil}, - keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d6", "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "00" + "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "01", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, + keyPair{addressToPubKeyHex(addr6, t, d) + "000370d6", txidB2T1 + "00" + txidB2T2 + "01", nil}, + keyPair{addressToPubKeyHex(addr7, t, d) + "000370d6", txidB2T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr8, t, d) + "000370d6", txidB2T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr9, t, d) + "000370d6", txidB2T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d6", txidB2T1 + "01", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d6", txidB2T1 + "03", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d6", txidB2T3 + "00" + txidB2T3 + "01", nil}, + keyPair{addressToPubKeyHex(addrA, t, d) + "000370d6", txidB2T4 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d6", txidB2T2 + "03", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + txidB1T1, + "00" + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + spentAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), nil, }, keyPair{ - "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + txidB1T2, + "00" + "03" + + spentAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + spentAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + spentAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), nil, }, keyPair{ - "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", - }) - }, + txidB2T1, + "02" + + inputAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + inputAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + + "02" + + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), + nil, }, keyPair{ - "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", + txidB2T2, + "02" + + inputAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + inputAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + "02" + + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), + nil, + }, + keyPair{ + txidB2T3, + "01" + + inputAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + + "01" + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), + nil, + }, + keyPair{ + txidB2T4, + "01" + inputAddressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero) + + "02" + + addressToPubKeyHexWithLength(addrA, t, d) + bigintToHex(satB2T4AA) + + addressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero), nil, }, }); err != nil { @@ -390,20 +518,30 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockAddresses, []keyPair{ - keyPair{"000370d6", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "00", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "00", - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02" + "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", - }) - }, + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "02" + bigintToHex(satB1T1A2) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "02" + bigintToHex(satB1T2A3) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "02" + bigintToHex(satB1T2A4) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "02" + bigintToHex(satB1T2A5) + bigintToHex(satB2T3A5), nil}, + keyPair{addressToPubKeyHex(addr6, t, d), "02" + bigintToHex(satB2T1A6) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr7, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T1A7), nil}, + keyPair{addressToPubKeyHex(addr8, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A8), nil}, + keyPair{addressToPubKeyHex(addr9, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A9), nil}, + keyPair{addressToPubKeyHex(addrA, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T4AA), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfBlockTxs, []keyPair{ + keyPair{ + "000370d6", + txidB2T1 + "02" + txidB1T2 + "00" + txidB1T1 + "02" + + txidB2T2 + "02" + txidB2T1 + "00" + txidB1T2 + "02" + + txidB2T3 + "01" + txidB1T2 + "04" + + txidB2T4 + "01" + "0000000000000000000000000000000000000000000000000000000000000000" + "00", + nil, }, }); err != nil { { @@ -474,19 +612,16 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality // 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using blockaddresses column -// 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error +// 6) Disconnect the block 2 using BlockTxs column +// 7) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ - BitcoinParser: &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, - Params: btc.GetChainParams("test"), - }, + BitcoinParser: bitcoinTestnetParser(), }) defer closeAndDestroyRocksDB(t, d) - // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column + // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column block1 := getTestUTXOBlock1(t, d) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) @@ -501,19 +636,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { verifyAfterUTXOBlock2(t, d) // get transactions for various addresses / low-high ranges - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225493, 225493, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, + verifyGetTransactions(t, d, addr2, 225493, 225493, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225494, 1000000, []txidVoutOutput{ - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 225494, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 500000, 1000000, []txidVoutOutput{}, nil) - verifyGetTransactions(t, d, "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", 0, true}, + verifyGetTransactions(t, d, addr2, 500000, 1000000, []txidVoutOutput{}, nil) + verifyGetTransactions(t, d, addr8, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T2, 0, true}, }, nil) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch")) @@ -561,13 +696,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } verifyAfterUTXOBlock2(t, d) + // try to disconnect both blocks, however only the last one is kept, it is not possible + err = d.DisconnectBlockRangeUTXO(225493, 225494) + if err == nil || err.Error() != "Cannot disconnect blocks with height 225493 and lower. It is necessary to rebuild index." { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs // and that the cached tx is removed - err = d.DisconnectBlockRange(225494, 225494) + err = d.DisconnectBlockRangeUTXO(225494, 225494) if err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, true) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { @@ -575,150 +716,273 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } } + // connect block again and verify the state of db + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) } -func Test_findAndRemoveUnspentAddr(t *testing.T) { - type args struct { - unspentAddrs string - vout uint32 +func Test_BulkConnect_UTXO(t *testing.T) { + d := setupRocksDB(t, &testBitcoinParser{ + BitcoinParser: bitcoinTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + bc, err := d.InitBulkConnect() + if err != nil { + t.Fatal(err) } + + if d.is.DbState != common.DbStateInconsistent { + t.Fatal("DB not in DbStateInconsistent") + } + + if err := bc.ConnectBlock(getTestUTXOBlock1(t, d), false); err != nil { + t.Fatal(err) + } + if err := checkColumn(d, cfBlockTxs, []keyPair{}); err != nil { + { + t.Fatal(err) + } + } + + if err := bc.ConnectBlock(getTestUTXOBlock2(t, d), true); err != nil { + t.Fatal(err) + } + + if err := bc.Close(); err != nil { + t.Fatal(err) + } + + if d.is.DbState != common.DbStateOpen { + t.Fatal("DB not in DbStateOpen") + } + + verifyAfterUTXOBlock2(t, d) +} + +func Test_packBigint_unpackBigint(t *testing.T) { + bigbig1, _ := big.NewInt(0).SetString("123456789123456789012345", 10) + bigbig2, _ := big.NewInt(0).SetString("12345678912345678901234512389012345123456789123456789012345123456789123456789012345", 10) + bigbigbig := big.NewInt(0) + bigbigbig.Mul(bigbig2, bigbig2) + bigbigbig.Mul(bigbigbig, bigbigbig) + bigbigbig.Mul(bigbigbig, bigbigbig) tests := []struct { - name string - args args - want string - want2 string + name string + bi *big.Int + buf []byte + toobiglen int }{ { - name: "3", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 3, - }, - want: "64635167006868", - want2: "029c0010517a0115887452870212709393588893935687040e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", + name: "0", + bi: big.NewInt(0), + buf: make([]byte, maxPackedBigintBytes), }, { - name: "10", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 10, - }, - want: "61", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112", + name: "1", + bi: big.NewInt(1), + buf: make([]byte, maxPackedBigintBytes), }, { - name: "not there", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 11, - }, - want: "", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", + name: "54321", + bi: big.NewInt(54321), + buf: make([]byte, 249), + }, + { + name: "12345678", + bi: big.NewInt(12345678), + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "123456789123456789", + bi: big.NewInt(123456789123456789), + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "bigbig1", + bi: bigbig1, + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "bigbig2", + bi: bigbig2, + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "bigbigbig", + bi: bigbigbig, + buf: make([]byte, maxPackedBigintBytes), + toobiglen: 242, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.unspentAddrs) - if err != nil { - panic(err) - } - got, got2 := findAndRemoveUnspentAddr(b, tt.args.vout) - h := hex.EncodeToString(got) - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("findAndRemoveUnspentAddr() got = %v, want %v", h, tt.want) - } - h2 := hex.EncodeToString(got2) - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("findAndRemoveUnspentAddr() got2 = %v, want %v", h2, tt.want2) + // packBigint + got := packBigint(tt.bi, tt.buf) + if tt.toobiglen == 0 { + // create buffer that we expect + bb := tt.bi.Bytes() + want := append([]byte(nil), byte(len(bb))) + want = append(want, bb...) + if got != len(want) { + t.Errorf("packBigint() = %v, want %v", got, len(want)) + } + for i := 0; i < got; i++ { + if tt.buf[i] != want[i] { + t.Errorf("packBigint() buf = %v, want %v", tt.buf[:got], want) + break + } + } + // unpackBigint + got1, got2 := unpackBigint(tt.buf) + if got2 != len(want) { + t.Errorf("unpackBigint() = %v, want %v", got2, len(want)) + } + if tt.bi.Cmp(&got1) != 0 { + t.Errorf("unpackBigint() = %v, want %v", got1, tt.bi) + } + } else { + if got != tt.toobiglen { + t.Errorf("packBigint() = %v, want toobiglen %v", got, tt.toobiglen) + } } }) } } -type hexoutpoint struct { - txID string - vout int32 +func addressToOutput(addr string, parser *btc.BitcoinParser) []byte { + b, err := parser.AddressToOutputScript(addr) + if err != nil { + panic(err) + } + return b } -func Test_unpackBlockAddresses(t *testing.T) { - d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}}) - defer closeAndDestroyRocksDB(t, d) - type args struct { - buf string - } +func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { + parser := bitcoinTestnetParser() tests := []struct { - name string - args args - want []string - want2 [][]hexoutpoint - wantErr bool + name string + hex string + data *txAddresses }{ { name: "1", - args: args{"029c0010517a011588745287047c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d250000b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa38400612709393588893935687000e64635167006868000e7651935188008702effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac7502"}, - want: []string{"9c", "517a011588745287", "709393588893935687", "64635167006868", "76519351880087"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 0}, - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 3}, + hex: "0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + data: &txAddresses{ + inputs: []txInput{ + { + addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + valueSat: *big.NewInt(0), + }, + { + addrID: addressToOutput("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), + valueSat: *big.NewInt(1234123421342341234), + }, }, - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + outputs: []txOutput{ + { + addrID: addressToOutput("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + valueSat: *big.NewInt(1), + spent: true, + }, }, }, }, { - name: "1", - args: args{"3276A914B434EB0C1A3B7A02E8A29CC616E791EF1E0BF51F88AC003276A9143F8BA3FDA3BA7B69F5818086E12223C6DD25E3C888AC003276A914A08EAE93007F22668AB5E4A9C83C8CD1C325E3E088AC02EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75003276A9148BDF0AA3C567AA5975C2E61321B8BEBBE7293DF688AC0200B2C06055E5E90E9C82BD4181FDE310104391A7FA4F289B1704E5D90CAA3840022EA9144A21DB08FB6882CB152E1FF06780A430740F77048702EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75023276A914CCAAAF374E1B06CB83118453D102587B4273D09588AC003276A9148D802C045445DF49613F6A70DDD2E48526F3701F88AC00"}, - want: []string{"76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac", "76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac", "76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac", "76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac", "a9144a21db08fb6882cb152e1ff06780a430740f770487", "76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac", "76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 0}, + name: "2", + hex: "0317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + data: &txAddresses{ + inputs: []txInput{ + { + addrID: addressToOutput("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + valueSat: *big.NewInt(9011000000), + }, + { + addrID: addressToOutput("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + valueSat: *big.NewInt(8011000000), + }, + { + addrID: addressToOutput("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + valueSat: *big.NewInt(7011000000), + }, }, - []hexoutpoint{ - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1}, + outputs: []txOutput{ + { + addrID: addressToOutput("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + valueSat: *big.NewInt(5011000000), + spent: true, + }, + { + addrID: addressToOutput("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + valueSat: *big.NewInt(6011000000), + }, + { + addrID: addressToOutput("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + valueSat: *big.NewInt(7011000000), + spent: true, + }, + { + addrID: addressToOutput("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + valueSat: *big.NewInt(999900000), + }, + { + addrID: addressToOutput("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + valueSat: *big.NewInt(5000000000), + spent: true, + }, }, - []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + }, + }, + { + name: "empty address", + hex: "01000204d2020002162e010162", + data: &txAddresses{ + inputs: []txInput{ + { + addrID: []byte{}, + valueSat: *big.NewInt(1234), + }, }, - []hexoutpoint{}, - []hexoutpoint{}, + outputs: []txOutput{ + { + addrID: []byte{}, + valueSat: *big.NewInt(5678), + }, + { + addrID: []byte{}, + valueSat: *big.NewInt(98), + spent: true, + }, + }, + }, + }, + { + name: "empty", + hex: "0000", + data: &txAddresses{ + inputs: []txInput{}, + outputs: []txOutput{}, }, }, } + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.buf) - if err != nil { - panic(err) + b := packTxAddresses(tt.data, buf, varBuf) + hex := hex.EncodeToString(b) + if !reflect.DeepEqual(hex, tt.hex) { + t.Errorf("packTxAddresses() = %v, want %v", hex, tt.hex) } - got, got2, err := d.unpackBlockAddresses(b) - if (err != nil) != tt.wantErr { - t.Errorf("unpackBlockAddresses() error = %v, wantErr %v", err, tt.wantErr) + got1, err := unpackTxAddresses(b) + if err != nil { + t.Errorf("unpackTxAddresses() error = %v", err) return } - h := make([]string, len(got)) - for i, g := range got { - h[i] = hex.EncodeToString(g) - } - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h, tt.want) - } - h2 := make([][]hexoutpoint, len(got2)) - for i, g := range got2 { - ho := make([]hexoutpoint, len(g)) - for j, o := range g { - ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.vout} - } - h2[i] = ho - } - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h2, tt.want2) + if !reflect.DeepEqual(got1, tt.data) { + t.Errorf("unpackTxAddresses() = %+v, want %+v", got1, tt.data) } }) } diff --git a/db/sync.go b/db/sync.go index a664e94f..32bc143d 100644 --- a/db/sync.go +++ b/db/sync.go @@ -213,17 +213,26 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { writeBlockDone := make(chan struct{}) writeBlockWorker := func() { defer close(writeBlockDone) + bc, err := w.db.InitBulkConnect() + if err != nil { + glog.Error("sync: InitBulkConnect error ", err) + } lastBlock := lower - 1 + keep := uint32(w.chain.GetChainParser().KeepBlockAddresses()) for b := range bch { if lastBlock+1 != b.Height { glog.Error("writeBlockWorker skipped block, last connected block", lastBlock, ", new block ", b.Height) } - err := w.db.ConnectBlock(b) + err := bc.ConnectBlock(b, b.Height+keep > higher) if err != nil { glog.Error("writeBlockWorker ", b.Height, " ", b.Hash, " error ", err) } lastBlock = b.Height } + err = bc.Close() + if err != nil { + glog.Error("sync: bulkconnect.Close error ", err) + } glog.Info("WriteBlock exiting...") } getBlockWorker := func(i int) { @@ -276,6 +285,7 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { } go writeBlockWorker() var hash string + start := time.Now() ConnectLoop: for h := lower; h <= higher; { select { @@ -292,7 +302,8 @@ ConnectLoop: } hch <- hashHeight{hash, h} if h > 0 && h%1000 == 0 { - glog.Info("connecting block ", h, " ", hash) + glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start)) + start = time.Now() } h++ } @@ -346,25 +357,23 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { } // DisconnectBlocks removes all data belonging to blocks in range lower-higher, -// using block data from blockchain, if they are available, -// otherwise doing full scan func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) - // if the chain uses Block to Addresses mapping, always use DisconnectBlockRange - if w.chain.GetChainParser().KeepBlockAddresses() > 0 { - return w.db.DisconnectBlockRange(lower, higher) + // if the chain is UTXO, always use DisconnectBlockRange + if w.chain.GetChainParser().IsUTXOChain() { + return w.db.DisconnectBlockRangeUTXO(lower, higher) } blocks := make([]*bchain.Block, len(hashes)) var err error - // get all blocks first to see if we can avoid full scan + // try to get all blocks first to see if we can avoid full scan for i, hash := range hashes { blocks[i], err = w.chain.GetBlock(hash, 0) if err != nil { // cannot get a block, we must do full range scan - return w.db.DisconnectBlockRange(lower, higher) + return w.db.DisconnectBlockRangeNonUTXO(lower, higher) } } - // then disconnect one after another + // got all blocks to be disconnected, disconnect them one after another for i, block := range blocks { glog.Info("Disconnecting block ", (int(higher) - i), " ", block.Hash) if err = w.db.DisconnectBlock(block); err != nil { diff --git a/server/public.go b/server/public.go index 54555325..acafd08c 100644 --- a/server/public.go +++ b/server/public.go @@ -113,8 +113,10 @@ func formatUnixTime(ut int64) string { return time.Unix(ut, 0).Format(time.RFC1123) } -func formatAmount(a float64) string { - return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%0.8f", a), "0"), ".") +// for now return the string as it is +// in future could be used to do coin specific formatting +func formatAmount(a string) string { + return a } // Run starts the server diff --git a/server/socketio.go b/server/socketio.go index 46c0a563..74fbd67b 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -7,6 +7,7 @@ import ( "blockbook/db" "encoding/json" "net/http" + "strconv" "strings" "time" @@ -236,11 +237,11 @@ type txInputs struct { // ScriptAsm *string `json:"scriptAsm"` Sequence int64 `json:"sequence"` Address *string `json:"address"` - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` } type txOutputs struct { - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` Script *string `json:"script"` // ScriptAsm *string `json:"scriptAsm"` SpentTxID *string `json:"spentTxId,omitempty"` @@ -267,7 +268,7 @@ type resTx struct { type addressHistoryItem struct { Addresses map[string]addressHistoryIndexes `json:"addresses"` - Satoshis int64 `json:"satoshis"` + Satoshis string `json:"satoshis"` Confirmations int `json:"confirmations"` Tx resTx `json:"tx"` } @@ -329,7 +330,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r for _, vout := range tx.Vout { aoh := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.String(), Script: &aoh, } if vout.Address != nil { @@ -452,6 +453,7 @@ func unmarshalEstimateSmartFee(params []byte) (blocks int, conservative bool, er } type resultEstimateSmartFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -460,7 +462,7 @@ func (s *SocketIoServer) estimateSmartFee(blocks int, conservative bool) (res re if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -479,6 +481,7 @@ func unmarshalEstimateFee(params []byte) (blocks int, err error) { } type resultEstimateFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -487,7 +490,7 @@ func (s *SocketIoServer) estimateFee(blocks int) (res resultEstimateFee, err err if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -588,7 +591,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai a := vout.Address.String() ai.Address = &a } - ai.Satoshis = int64(vout.Value*1E8 + 0.5) + ai.Satoshis = vout.ValueSat.String() } } hi = append(hi, ai) @@ -596,7 +599,7 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai for _, vout := range tx.Vout { aos := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.String(), Script: &aos, } if vout.Address != nil {