diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ffb9789..fefff9a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,9 +81,8 @@ In section *blockbook* update information how to build and configure Blockbook s Update *package_maintainer* and *package_maintainer_email* options in section *meta*. -Execute script *contrib/scripts/check-ports.go* that will check mandatory ports and uniquity of registered ports. - -Execute script *contrib/scripts/generate-port-registry.go* that will update *docs/ports.md*. +Execute script *go run contrib/scripts/check-and-generate-port-registry.go -w* that checks mandatory ports and +uniqueness of ports and updates registry of ports *docs/ports.md*. Now you can try generate package definitions as described above in order to check outputs. diff --git a/Gopkg.lock b/Gopkg.lock index 3dd541e8..f9a70817 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -49,12 +49,6 @@ packages = ["ripemd160"] revision = "53f62d9b43e87a6c56975cf862af7edf33a8d0df" -[[projects]] - branch = "master" - name = "github.com/cpacia/bchutil" - packages = ["."] - revision = "12e86f41eb040d3b85b5d8e3a3a4bed035517c52" - [[projects]] name = "github.com/dchest/blake256" packages = ["."] @@ -67,6 +61,12 @@ revision = "1d4478f51bed434f1dadf96dcd9b43aabac66795" version = "v1.7" +[[projects]] + branch = "master" + name = "github.com/erikdubbelboer/gspt" + packages = ["."] + revision = "e39e726e09cc23d1ccf13b36ce10dbdb4a4510e0" + [[projects]] name = "github.com/ethereum/go-ethereum" packages = [".","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","ethdb","log","metrics","p2p/netutil","params","rlp","rpc","trie"] @@ -121,6 +121,18 @@ revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" version = "v1.2.0" +[[projects]] + branch = "master" + name = "github.com/jakm/bchutil" + packages = ["."] + revision = "9e4fc13b082c87967b0befbcbad6fe5a5aa5ac1a" + +[[projects]] + branch = "master" + name = "github.com/jakm/btcutil" + packages = [".","base58","bech32","chaincfg","txscript"] + revision = "a45c5a6a9cb32f0caecb14a4e5a8f82640be1f39" + [[projects]] branch = "master" name = "github.com/juju/errors" @@ -244,6 +256,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "03817720a4b1b5011cc64018a1344ed3fb4d8228af41c86856069f57851b3819" + inputs-digest = "98fb9a6252acd86d1c0a4b09a42e34673c2ccd6327caf49ed85bcee573601d95" solver-name = "gps-cdcl" solver-version = 1 diff --git a/api/types.go b/api/types.go index b605bd30..32280427 100644 --- a/api/types.go +++ b/api/types.go @@ -1,65 +1,138 @@ package api +import ( + "blockbook/bchain" + "blockbook/common" + "blockbook/db" + "math/big" + "time" +) + +const BlockbookAbout = "Blockbook - blockchain indexer for TREZOR wallet https://trezor.io/. Do not use for any other purpose." + +type ApiError struct { + Text string + Public bool +} + +func (e *ApiError) Error() string { + return e.Text +} + +func NewApiError(s string, public bool) error { + return &ApiError{ + Text: s, + Public: public, + } +} + type ScriptSig struct { Hex string `json:"hex"` Asm string `json:"asm,omitempty"` } type Vin struct { - Txid string `json:"txid"` - Vout uint32 `json:"vout"` - Sequence int64 `json:"sequence,omitempty"` - N int `json:"n"` - ScriptSig ScriptSig `json:"scriptSig"` - Addr string `json:"addr"` - ValueSat int64 `json:"valueSat"` - Value float64 `json:"value"` + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Sequence int64 `json:"sequence,omitempty"` + N int `json:"n"` + ScriptSig ScriptSig `json:"scriptSig"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` } type ScriptPubKey struct { - Hex string `json:"hex"` - Asm string `json:"asm,omitempty"` - Addresses []string `json:"addresses"` - Type string `json:"type,omitempty"` + Hex string `json:"hex"` + Asm string `json:"asm,omitempty"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Type string `json:"type,omitempty"` } type Vout struct { - Value float64 `json:"value"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` N int `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` + Spent bool `json:"-"` SpentTxID string `json:"spentTxId,omitempty"` SpentIndex int `json:"spentIndex,omitempty"` SpentHeight int `json:"spentHeight,omitempty"` } type Tx struct { - Txid string `json:"txid"` - Version int32 `json:"version,omitempty"` - Locktime uint32 `json:"locktime,omitempty"` - Vin []Vin `json:"vin"` - Vout []Vout `json:"vout"` - Blockhash string `json:"blockhash,omitempty"` - Blockheight int `json:"blockheight"` - Confirmations uint32 `json:"confirmations"` - Time int64 `json:"time,omitempty"` - Blocktime int64 `json:"blocktime"` - ValueOut float64 `json:"valueOut"` - Size int `json:"size,omitempty"` - ValueIn float64 `json:"valueIn"` - Fees float64 `json:"fees"` - WithSpends bool `json:"withSpends,omitempty"` + Txid string `json:"txid"` + Version int32 `json:"version,omitempty"` + Locktime uint32 `json:"locktime,omitempty"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` + Blockhash string `json:"blockhash,omitempty"` + Blockheight int `json:"blockheight"` + Confirmations uint32 `json:"confirmations"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime"` + ValueOut string `json:"valueOut"` + Size int `json:"size,omitempty"` + ValueIn string `json:"valueIn"` + Fees string `json:"fees"` + Hex string `json:"hex"` +} + +type Paging struct { + Page int `json:"page"` + TotalPages int `json:"totalPages"` + ItemsOnPage int `json:"itemsOnPage"` } type Address struct { - AddrStr string `json:"addrStr"` - Balance float64 `json:"balance"` - BalanceSat int64 `json:"balanceSat"` - TotalReceived float64 `json:"totalReceived"` - TotalReceivedSat int64 `json:"totalReceivedSat"` - TotalSent float64 `json:"totalSent"` - TotalSentSat int64 `json:"totalSentSat"` - UnconfirmedBalance float64 `json:"unconfirmedBalance"` - UnconfirmedBalanceSat int64 `json:"unconfirmedBalanceSat"` - UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` - TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"transactions"` + Paging + AddrStr string `json:"addrStr"` + Balance string `json:"balance"` + TotalReceived string `json:"totalReceived"` + TotalSent string `json:"totalSent"` + UnconfirmedBalance string `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*Tx `json:"txs,omitempty"` + Txids []string `json:"transactions,omitempty"` +} + +type Blocks struct { + Paging + Blocks []db.BlockInfo `json:"blocks"` +} + +type Block struct { + Paging + bchain.BlockInfo + TxCount int `json:"TxCount"` + Transactions []*Tx `json:"txs,omitempty"` +} + +type BlockbookInfo struct { + Coin string `json:"coin"` + Host string `json:"host"` + Version string `json:"version"` + GitCommit string `json:"gitcommit"` + BuildTime string `json:"buildtime"` + SyncMode bool `json:"syncMode"` + InitialSync bool `json:"initialsync"` + InSync bool `json:"inSync"` + BestHeight uint32 `json:"bestHeight"` + LastBlockTime time.Time `json:"lastBlockTime"` + InSyncMempool bool `json:"inSyncMempool"` + LastMempoolTime time.Time `json:"lastMempoolTime"` + MempoolSize int `json:"mempoolSize"` + DbSize int64 `json:"dbSize"` + DbSizeFromColumns int64 `json:"dbSizeFromColumns,omitempty"` + DbColumns []common.InternalStateColumn `json:"dbColumns,omitempty"` + About string `json:"about"` +} + +type SystemInfo struct { + Blockbook *BlockbookInfo `json:"blockbook"` + Backend *bchain.ChainInfo `json:"backend"` } diff --git a/api/worker.go b/api/worker.go index 4ad421d3..2dd6cb89 100644 --- a/api/worker.go +++ b/api/worker.go @@ -4,12 +4,16 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "bytes" + "fmt" + "math/big" + "strconv" + "time" "github.com/golang/glog" + "github.com/juju/errors" ) -const txsOnPage = 30 - // Worker is handle to api worker type Worker struct { db *db.RocksDB @@ -31,20 +35,89 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is return w, nil } +func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescriptor, []string, bool, error) { + addrDesc, err := w.chainParser.GetAddrDescFromVout(vout) + if err != nil { + return nil, nil, false, err + } + a, s, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + return addrDesc, a, s, err +} + +// setSpendingTxToVout is helper function, that finds transaction that spent given output and sets it to the output +// there is not an index, it must be found using addresses -> txaddresses -> tx +func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height, bestheight uint32) error { + err := w.db.GetAddrDescTransactions(vout.ScriptPubKey.AddrDesc, height, ^uint32(0), func(t string, index uint32, isOutput bool) error { + if isOutput == false { + tsp, err := w.db.GetTxAddresses(t) + if err != nil { + glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses") + } else { + if len(tsp.Inputs) > int(index) { + if tsp.Inputs[index].ValueSat.Cmp(&vout.ValueSat) == 0 { + spentTx, spentHeight, err := w.txCache.GetTransaction(t, bestheight) + if err != nil { + glog.Warning("Tx ", t, ": not found") + } else { + if len(spentTx.Vin) > int(index) { + if spentTx.Vin[index].Txid == txid { + vout.SpentTxID = t + vout.SpentHeight = int(spentHeight) + vout.SpentIndex = int(index) + return &db.StopIteration{} + } + } + } + } + } + } + } + return nil + }) + return err +} + +// GetSpendingTxid returns transaction id of transaction that spent given output +func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) { + start := time.Now() + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return "", err + } + tx, err := w.GetTransaction(txid, bestheight, false) + if err != nil { + return "", err + } + if n >= len(tx.Vout) || n < 0 { + return "", NewApiError(fmt.Sprintf("Passed incorrect vout index %v for tx %v, len vout %v", n, tx.Txid, len(tx.Vout)), false) + } + err = w.setSpendingTxToVout(&tx.Vout[n], tx.Txid, uint32(tx.Blockheight), bestheight) + if err != nil { + return "", err + } + glog.Info("GetSpendingTxid ", txid, " ", n, " finished in ", time.Since(start)) + return tx.Vout[n].SpentTxID, nil +} + // GetTransaction reads transaction data from txid -func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) (*Tx, error) { +func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool) (*Tx, error) { + start := time.Now() bchainTx, height, err := w.txCache.GetTransaction(txid, bestheight) if err != nil { - return nil, err + return nil, NewApiError(fmt.Sprintf("Tx not found, %v", err), true) + } + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) } var blockhash string if bchainTx.Confirmations > 0 { blockhash, err = w.db.GetBlockHash(height) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetBlockHash %v", height) } } - var valIn, valOut, fees float64 + var valInSat, valOutSat, feesSat big.Int vins := make([]Vin, len(bchainTx.Vin)) for i := range bchainTx.Vin { bchainVin := &bchainTx.Vin[i] @@ -55,20 +128,43 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) vin.ScriptSig.Hex = bchainVin.ScriptSig.Hex // bchainVin.Txid=="" is coinbase transaction if bchainVin.Txid != "" { - otx, _, err := w.txCache.GetTransaction(bchainVin.Txid, bestheight) + // load spending addresses from TxAddresses + tas, err := w.db.GetTxAddresses(bchainVin.Txid) if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) } - if len(otx.Vout) > int(vin.Vout) { - vout := &otx.Vout[vin.Vout] - vin.Value = vout.Value - valIn += vout.Value - vin.ValueSat = int64(vout.Value*1E8 + 0.5) - if vout.Address != nil { - a := vout.Address.String() - vin.Addr = a + if tas == nil { + // mempool transactions are not in TxAddresses but confirmed should be there, log a problem + if bchainTx.Confirmations > 0 { + glog.Warning("DB inconsistency: tx ", bchainVin.Txid, ": not found in txAddresses") + } + // try to load from backend + otx, _, err := w.txCache.GetTransaction(bchainVin.Txid, bestheight) + if err != nil { + return nil, errors.Annotatef(err, "txCache.GetTransaction %v", bchainVin.Txid) + } + if len(otx.Vout) > int(vin.Vout) { + vout := &otx.Vout[vin.Vout] + vin.ValueSat = vout.ValueSat + vin.AddrDesc, vin.Addresses, vin.Searchable, err = w.getAddressesFromVout(vout) + if err != nil { + glog.Errorf("getAddressesFromVout error %v, vout %+v", err, vout) + } + } + } else { + if len(tas.Outputs) > int(vin.Vout) { + output := &tas.Outputs[vin.Vout] + vin.ValueSat = output.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + vin.AddrDesc = output.AddrDesc + vin.Addresses, vin.Searchable, err = output.Addresses(w.chainParser) + if err != nil { + glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) + } } } + vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + valInSat.Add(&valInSat, &vin.ValueSat) } } vouts := make([]Vout, len(bchainTx.Vout)) @@ -76,18 +172,25 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) bchainVout := &bchainTx.Vout[i] vout := &vouts[i] vout.N = i - vout.Value = bchainVout.Value - valOut += bchainVout.Value + vout.ValueSat = bchainVout.ValueSat + vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) + valOutSat.Add(&valOutSat, &bchainVout.ValueSat) vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex - vout.ScriptPubKey.Addresses = bchainVout.ScriptPubKey.Addresses - if spendingTx { - // TODO + vout.ScriptPubKey.AddrDesc, vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = w.getAddressesFromVout(bchainVout) + if ta != nil { + vout.Spent = ta.Outputs[i].Spent + if spendingTxs && vout.Spent { + err = w.setSpendingTxToVout(vout, bchainTx.Txid, height, bestheight) + if err != nil { + glog.Errorf("setSpendingTxToVout error %v, %v, output %v", err, vout.ScriptPubKey.AddrDesc, vout.N) + } + } } } // for coinbase transactions valIn is 0 - fees = valIn - valOut - if fees < 0 { - fees = 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) } // for now do not return size, we would have to compute vsize of segwit transactions // size:=len(bchainTx.Hex) / 2 @@ -96,25 +199,28 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTx bool) Blockheight: int(height), Blocktime: bchainTx.Blocktime, Confirmations: bchainTx.Confirmations, - Fees: fees, + Fees: w.chainParser.AmountToDecimalString(&feesSat), Locktime: bchainTx.LockTime, - WithSpends: spendingTx, Time: bchainTx.Time, Txid: bchainTx.Txid, - ValueIn: valIn, - ValueOut: valOut, + ValueIn: w.chainParser.AmountToDecimalString(&valInSat), + ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), Version: bchainTx.Version, + Hex: bchainTx.Hex, Vin: vins, Vout: vouts, } + if spendingTxs { + glog.Info("GetTransaction ", txid, " finished in ", time.Since(start)) + } return r, nil } -func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) { +func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool) ([]string, error) { var err error txids := make([]string, 0) if !mempool { - err = s.db.GetTransactions(address, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { + err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { txids = append(txids, txid) return nil }) @@ -122,7 +228,7 @@ func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return nil, err } } else { - m, err := s.chain.GetMempoolTransactions(address) + m, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) if err != nil { return nil, err } @@ -131,26 +237,24 @@ func (s *Worker) getAddressTxids(address string, mempool bool) ([]string, error) return txids, nil } -func (t *Tx) getAddrVoutValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVoutValue(addrDesc bchain.AddressDescriptor) *big.Int { + var val big.Int for _, vout := range t.Vout { - for _, a := range vout.ScriptPubKey.Addresses { - if a == addrID { - val += vout.Value - } + if bytes.Equal(vout.ScriptPubKey.AddrDesc, addrDesc) { + val.Add(&val, &vout.ValueSat) } } - return val + return &val } -func (t *Tx) getAddrVinValue(addrID string) float64 { - var val float64 +func (t *Tx) getAddrVinValue(addrDesc bchain.AddressDescriptor) *big.Int { + var val big.Int for _, vin := range t.Vin { - if vin.Addr == addrID { - val += vin.Value + if bytes.Equal(vin.AddrDesc, addrDesc) { + val.Add(&val, &vin.ValueSat) } } - return val + return &val } // UniqueTxidsInReverse reverts the order of transactions (so that newest are first) and removes duplicate transactions @@ -169,75 +273,320 @@ func UniqueTxidsInReverse(txids []string) []string { return ut[i:] } -// GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddress(addrID string, page int) (*Address, error) { - glog.Info(addrID, " start") - txc, err := w.getAddressTxids(addrID, false) - txc = UniqueTxidsInReverse(txc) - if err != nil { - return nil, err +func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockInfo, bestheight uint32) *Tx { + var err error + var valInSat, valOutSat, feesSat big.Int + vins := make([]Vin, len(ta.Inputs)) + for i := range ta.Inputs { + tai := &ta.Inputs[i] + vin := &vins[i] + vin.N = i + vin.ValueSat = tai.ValueSat + vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) + valInSat.Add(&valInSat, &vin.ValueSat) + vin.Addresses, vin.Searchable, err = tai.Addresses(w.chainParser) + if err != nil { + glog.Errorf("tai.Addresses error %v, tx %v, input %v, tai %+v", err, txid, i, tai) + } } - txm, err := w.getAddressTxids(addrID, true) + vouts := make([]Vout, len(ta.Outputs)) + for i := range ta.Outputs { + tao := &ta.Outputs[i] + vout := &vouts[i] + vout.N = i + vout.ValueSat = tao.ValueSat + vout.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) + valOutSat.Add(&valOutSat, &vout.ValueSat) + vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = tao.Addresses(w.chainParser) + if err != nil { + glog.Errorf("tai.Addresses error %v, tx %v, output %v, tao %+v", err, txid, i, tao) + } + vout.Spent = tao.Spent + } + // for coinbase transactions valIn is 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) + } + r := &Tx{ + Blockhash: bi.Hash, + Blockheight: int(ta.Height), + Blocktime: bi.Time, + Confirmations: bestheight - ta.Height + 1, + Fees: w.chainParser.AmountToDecimalString(&feesSat), + Time: bi.Time, + Txid: txid, + ValueIn: w.chainParser.AmountToDecimalString(&valInSat), + ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), + Vin: vins, + Vout: vouts, + } + return r +} + +func computePaging(count, page, itemsOnPage int) (Paging, int, int, int) { + from := page * itemsOnPage + totalPages := (count - 1) / itemsOnPage + if totalPages < 0 { + totalPages = 0 + } + if from >= count { + page = totalPages + } + from = page * itemsOnPage + to := (page + 1) * itemsOnPage + if to > count { + to = count + } + return Paging{ + ItemsOnPage: itemsOnPage, + Page: page + 1, + TotalPages: totalPages + 1, + }, from, to, page +} + +// GetAddress computes address value and gets transactions for given address +func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids bool) (*Address, error) { + start := time.Now() + page-- + if page < 0 { + page = 0 + } + addrDesc, err := w.chainParser.GetAddrDescFromAddress(address) if err != nil { - return nil, err + return nil, NewApiError(fmt.Sprintf("Address not found, %v", err), true) + } + // ba can be nil if the address is only in mempool! + ba, err := w.db.GetAddrDescBalance(addrDesc) + if err != nil { + return nil, NewApiError(fmt.Sprintf("Address not found, %v", err), true) + } + // convert the address to the format defined by the parser + addresses, _, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + if len(addresses) == 1 { + address = addresses[0] + } + txc, err := w.getAddressTxids(addrDesc, false) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v false", address) + } + txc = UniqueTxidsInReverse(txc) + var txm []string + // if there are only unconfirmed transactions, ba is nil + if ba == nil { + ba = &db.AddrBalance{} + page = 0 + } + txm, err = w.getAddressTxids(addrDesc, true) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v true", address) + } + txm = UniqueTxidsInReverse(txm) + // check if the address exist + if len(txc)+len(txm) == 0 { + return nil, NewApiError("Address not found", true) } bestheight, _, err := w.db.GetBestBlock() if err != nil { - return nil, err + return nil, errors.Annotatef(err, "GetBestBlock") } - lc := len(txc) - if lc > txsOnPage { - lc = txsOnPage + pg, from, to, page := computePaging(len(txc), page, txsOnPage) + var txs []*Tx + var txids []string + if onlyTxids { + txids = make([]string, len(txm)+to-from) + } else { + txs = make([]*Tx, len(txm)+to-from) } - txs := make([]*Tx, len(txm)+lc) txi := 0 - var uBal, bal, totRecv, totSent float64 + // load mempool transactions + var uBalSat big.Int for _, tx := range txm { tx, err := w.GetTransaction(tx, bestheight, false) // mempool transaction may fail if err != nil { - glog.Error("GetTransaction ", tx, ": ", err) + glog.Error("GetTransaction in mempool ", tx, ": ", err) } else { - uBal = tx.getAddrVoutValue(addrID) - tx.getAddrVinValue(addrID) - txs[txi] = tx - txi++ - } - } - if page < 0 { - page = 0 - } - from := page * txsOnPage - if from > len(txc) { - from = 0 - } - to := from + txsOnPage - for i, tx := range txc { - tx, err := w.GetTransaction(tx, bestheight, false) - if err != nil { - return nil, err - } else { - totRecv += tx.getAddrVoutValue(addrID) - totSent += tx.getAddrVinValue(addrID) - if i >= from && i < to { - txs[txi] = tx + uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) + uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) + if page == 0 { + if onlyTxids { + txids[txi] = tx.Txid + } else { + txs[txi] = tx + } txi++ } } } - bal = totRecv - totSent - r := &Address{ - AddrStr: addrID, - Balance: bal, - BalanceSat: int64(bal*1E8 + 0.5), - TotalReceived: totRecv, - TotalReceivedSat: int64(totRecv*1E8 + 0.5), - TotalSent: totSent, - TotalSentSat: int64(totSent*1E8 + 0.5), - Transactions: txs[:txi], - TxApperances: len(txc), - UnconfirmedBalance: uBal, - UnconfirmedTxApperances: len(txm), + if len(txc) != int(ba.Txs) { + glog.Warning("DB inconsistency for address ", address, ": number of txs from column addresses ", len(txc), ", from addressBalance ", ba.Txs) } - glog.Info(addrID, " finished") + for i := from; i < to; i++ { + txid := txc[i] + if onlyTxids { + txids[txi] = txid + } else { + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + bi, err := w.db.GetBlockInfo(ta.Height) + if err != nil { + return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) + } + if bi == nil { + glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) + } + txi++ + } + if onlyTxids { + txids = txids[:txi] + } else { + txs = txs[:txi] + } + r := &Address{ + Paging: pg, + AddrStr: address, + Balance: w.chainParser.AmountToDecimalString(&ba.BalanceSat), + TotalReceived: w.chainParser.AmountToDecimalString(ba.ReceivedSat()), + TotalSent: w.chainParser.AmountToDecimalString(&ba.SentSat), + TxApperances: len(txc), + UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), + UnconfirmedTxApperances: len(txm), + Transactions: txs, + Txids: txids, + } + glog.Info("GetAddress ", address, " finished in ", time.Since(start)) return r, nil } + +// GetBlocks returns BlockInfo for blocks on given page +func (w *Worker) GetBlocks(page int, blocksOnPage int) (*Blocks, error) { + start := time.Now() + page-- + if page < 0 { + page = 0 + } + b, _, err := w.db.GetBestBlock() + bestheight := int(b) + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + pg, from, to, page := computePaging(bestheight+1, page, blocksOnPage) + r := &Blocks{Paging: pg} + r.Blocks = make([]db.BlockInfo, to-from) + for i := from; i < to; i++ { + bi, err := w.db.GetBlockInfo(uint32(bestheight - i)) + if err != nil { + return nil, err + } + r.Blocks[i-from] = *bi + } + glog.Info("GetBlocks page ", page, " finished in ", time.Since(start)) + return r, nil +} + +// GetBlock returns paged data about block +func (w *Worker) GetBlock(bid string, page int, txsOnPage int) (*Block, error) { + start := time.Now() + page-- + if page < 0 { + page = 0 + } + var hash string + height, err := strconv.Atoi(bid) + if err == nil && height < int(^uint32(0)) { + hash, err = w.db.GetBlockHash(uint32(height)) + } else { + hash = bid + } + bi, err := w.chain.GetBlockInfo(hash) + if err != nil { + if err == bchain.ErrBlockNotFound { + return nil, NewApiError("Block not found", true) + } + return nil, NewApiError(fmt.Sprintf("Block not found, %v", err), true) + } + dbi := &db.BlockInfo{ + Hash: bi.Hash, + Height: bi.Height, + Time: bi.Time, + } + txCount := len(bi.Txids) + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + pg, from, to, page := computePaging(txCount, page, txsOnPage) + glog.Info("GetBlock ", bid, ", page ", page, " finished in ", time.Since(start)) + txs := make([]*Tx, to-from) + txi := 0 + for i := from; i < to; i++ { + txid := bi.Txids[i] + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, dbi, bestheight) + txi++ + } + txs = txs[:txi] + bi.Txids = nil + return &Block{ + Paging: pg, + BlockInfo: *bi, + TxCount: txCount, + Transactions: txs, + }, nil +} + +// GetSystemInfo returns information about system +func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) { + start := time.Now() + ci, err := w.chain.GetChainInfo() + if err != nil { + return nil, errors.Annotatef(err, "GetChainInfo") + } + vi := common.GetVersionInfo() + ss, bh, st := w.is.GetSyncState() + ms, mt, msz := w.is.GetMempoolSyncState() + var dbc []common.InternalStateColumn + var dbs int64 + if internal { + dbc = w.is.GetAllDBColumnStats() + dbs = w.is.DBSizeTotal() + } + bi := &BlockbookInfo{ + Coin: w.is.Coin, + Host: w.is.Host, + Version: vi.Version, + GitCommit: vi.GitCommit, + BuildTime: vi.BuildTime, + SyncMode: w.is.SyncMode, + InitialSync: w.is.InitialSync, + InSync: ss, + BestHeight: bh, + LastBlockTime: st, + InSyncMempool: ms, + LastMempoolTime: mt, + MempoolSize: msz, + DbSize: w.db.DatabaseSizeOnDisk(), + DbSizeFromColumns: dbs, + DbColumns: dbc, + About: BlockbookAbout, + } + glog.Info("GetSystemInfo finished in ", time.Since(start)) + return &SystemInfo{bi, ci}, nil +} diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 8f7bc08a..48bb752e 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -3,27 +3,17 @@ package bchain import ( "encoding/hex" "encoding/json" + "math/big" + "strings" "github.com/gogo/protobuf/proto" "github.com/juju/errors" ) -type AddressFactoryFunc func(string) (Address, error) - // BaseParser implements data parsing/handling functionality base for all other parsers type BaseParser struct { - AddressFactory AddressFactoryFunc BlockAddressesToKeep int -} - -// AddressToOutputScript converts address to ScriptPubKey - currently not implemented -func (p *BaseParser) AddressToOutputScript(address string) ([]byte, error) { - return nil, errors.New("AddressToOutputScript: not implemented") -} - -// OutputScriptToAddresses converts ScriptPubKey to addresses - currently not implemented -func (p *BaseParser) OutputScriptToAddresses(script []byte) ([]string, error) { - return nil, errors.New("OutputScriptToAddresses: not implemented") + AmountDecimalPoint int } // ParseBlock parses raw block to our Block struct - currently not implemented @@ -36,7 +26,52 @@ func (p *BaseParser) ParseTx(b []byte) (*Tx, error) { return nil, errors.New("ParseTx: not implemented") } -// ParseTxFromJson parses JSON message containing transaction and returs Tx struct +const zeros = "0000000000000000000000000000000000000000" + +// AmountToBigInt converts amount in json.Number (string) to big.Int +// it uses string operations to avoid problems with rounding +func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { + var r big.Int + s := string(n) + i := strings.IndexByte(s, '.') + if i == -1 { + s = s + zeros[:p.AmountDecimalPoint] + } else { + z := p.AmountDecimalPoint - len(s) + i + 1 + if z > 0 { + s = s[:i] + s[i+1:] + zeros[:z] + } else { + s = s[:i] + s[i+1:len(s)+z] + } + } + if _, ok := r.SetString(s, 10); !ok { + return r, errors.New("AmountToBigInt: failed to convert") + } + return r, nil +} + +// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place +func (p *BaseParser) AmountToDecimalString(a *big.Int) string { + n := a.String() + var s string + if n[0] == '-' { + n = n[1:] + s = "-" + } + if len(n) <= p.AmountDecimalPoint { + n = zeros[:p.AmountDecimalPoint-len(n)+1] + n + } + i := len(n) - p.AmountDecimalPoint + ad := strings.TrimRight(n[i:], "0") + if len(ad) > 0 { + n = n[:i] + "." + ad + } else { + n = n[:i] + } + return s + n +} + +// ParseTxFromJson parses JSON message containing transaction and returns Tx struct func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { var tx Tx err := json.Unmarshal(msg, &tx) @@ -44,14 +79,14 @@ func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { return nil, err } - for i, vout := range tx.Vout { - if len(vout.ScriptPubKey.Addresses) == 1 { - a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) - if err != nil { - return nil, err - } - tx.Vout[i].Address = a + for i := range tx.Vout { + vout := &tx.Vout[i] + // convert vout.JsonValue to big.Int and clear it, it is only temporary value used for unmarshal + vout.ValueSat, err = p.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err } + vout.JsonValue = "" } return &tx, nil @@ -127,7 +162,7 @@ func (p *BaseParser) PackTx(tx *Tx, height uint32, blockTime int64) ([]byte, err Addresses: vo.ScriptPubKey.Addresses, N: vo.N, ScriptPubKeyHex: hex, - Value: vo.Value, + ValueSat: vo.ValueSat.Bytes(), } } pt := &ProtoTransaction{ @@ -176,20 +211,15 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } vout := make([]Vout, len(pt.Vout)) for i, pto := range pt.Vout { + var vs big.Int + vs.SetBytes(pto.ValueSat) vout[i] = Vout{ N: pto.N, ScriptPubKey: ScriptPubKey{ Addresses: pto.Addresses, Hex: hex.EncodeToString(pto.ScriptPubKeyHex), }, - Value: pto.Value, - } - if len(pto.Addresses) == 1 { - a, err := p.AddressFactory(pto.Addresses[0]) - if err != nil { - return nil, 0, err - } - vout[i].Address = a + ValueSat: vs, } } tx := Tx{ @@ -203,29 +233,3 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { } return &tx, pt.Height, nil } - -type baseAddress struct { - addr string -} - -func NewBaseAddress(addr string) (Address, error) { - return &baseAddress{addr: addr}, nil -} - -func (a baseAddress) String() string { - return a.addr -} - -func (a baseAddress) AreEqual(addr string) bool { - return a.String() == addr -} - -func (a baseAddress) InSlice(addrs []string) bool { - ea := a.String() - for _, addr := range addrs { - if ea == addr { - return true - } - } - return false -} diff --git a/bchain/baseparser_test.go b/bchain/baseparser_test.go new file mode 100644 index 00000000..cfb20a68 --- /dev/null +++ b/bchain/baseparser_test.go @@ -0,0 +1,56 @@ +package bchain + +import ( + "encoding/json" + "math/big" + "testing" +) + +func NewBaseParser(adp int) *BaseParser { + return &BaseParser{ + AmountDecimalPoint: adp, + } +} + +var amounts = []struct { + a *big.Int + s string + adp int + alternative string +}{ + {big.NewInt(123456789), "1.23456789", 8, "!"}, + {big.NewInt(2), "0.00000002", 8, "!"}, + {big.NewInt(300000000), "3", 8, "!"}, + {big.NewInt(498700000), "4.987", 8, "!"}, + {big.NewInt(567890), "0.00000000000056789", 18, "!"}, + {big.NewInt(-100000000), "-1", 8, "!"}, + {big.NewInt(-8), "-0.00000008", 8, "!"}, + {big.NewInt(-89012345678), "-890.12345678", 8, "!"}, + {big.NewInt(-12345), "-0.00012345", 8, "!"}, + {big.NewInt(12345678), "0.123456789012", 8, "0.12345678"}, // test of truncation of too many decimal places +} + +func TestBaseParser_AmountToDecimalString(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + if got := NewBaseParser(tt.adp).AmountToDecimalString(tt.a); got != tt.s && got != tt.alternative { + t.Errorf("BaseParser.AmountToDecimalString() = %v, want %v", got, tt.s) + } + }) + } +} + +func TestBaseParser_AmountToBigInt(t *testing.T) { + for _, tt := range amounts { + t.Run(tt.s, func(t *testing.T) { + got, err := NewBaseParser(tt.adp).AmountToBigInt(json.Number(tt.s)) + if err != nil { + t.Errorf("BaseParser.AmountToBigInt() error = %v", err) + return + } + if got.Cmp(tt.a) != 0 { + t.Errorf("BaseParser.AmountToBigInt() = %v, want %v", got, tt.a) + } + }) + } +} diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 95284d84..1348619d 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -5,10 +5,10 @@ import ( "blockbook/bchain/coins/btc" "fmt" - "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/txscript" - "github.com/btcsuite/btcutil" - "github.com/cpacia/bchutil" + "github.com/jakm/bchutil" + "github.com/jakm/btcutil" + "github.com/jakm/btcutil/chaincfg" + "github.com/jakm/btcutil/txscript" "github.com/schancel/cashaddr-converter/address" ) @@ -47,14 +47,14 @@ func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) (*BCashParser p := &BCashParser{ BitcoinParser: &btc.BitcoinParser{ BaseParser: &bchain.BaseParser{ - AddressFactory: func(addr string) (bchain.Address, error) { return newBCashAddress(addr, format) }, BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, Params: params, - OutputScriptToAddressesFunc: outputScriptToAddresses, }, AddressFormat: format, } + p.OutputScriptToAddressesFunc = p.outputScriptToAddresses return p, nil } @@ -78,13 +78,13 @@ func GetChainParams(chain string) *chaincfg.Params { return params } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *BCashParser) GetAddrIDFromAddress(address string) ([]byte, error) { - return p.AddressToOutputScript(address) +// GetAddrDescFromAddress returns internal address representation of given address +func (p *BCashParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { + return p.addressToOutputScript(address) } -// AddressToOutputScript converts bitcoin address to ScriptPubKey -func (p *BCashParser) AddressToOutputScript(address string) ([]byte, error) { +// addressToOutputScript converts bitcoin address to ScriptPubKey +func (p *BCashParser) addressToOutputScript(address string) ([]byte, error) { if isCashAddr(address) { da, err := bchutil.DecodeAddress(address, p.Params) if err != nil { @@ -123,68 +123,49 @@ func isCashAddr(addr string) bool { } // outputScriptToAddresses converts ScriptPubKey to bitcoin addresses -func outputScriptToAddresses(script []byte, params *chaincfg.Params) ([]string, error) { - a, err := bchutil.ExtractPkScriptAddrs(script, params) +func (p *BCashParser) outputScriptToAddresses(script []byte) ([]string, bool, error) { + a, err := bchutil.ExtractPkScriptAddrs(script, p.Params) if err != nil { - return nil, err - } - return []string{a.EncodeAddress()}, nil -} - -type bcashAddress struct { - addr string -} - -func newBCashAddress(addr string, format AddressFormat) (*bcashAddress, error) { - if isCashAddr(addr) && format == CashAddr { - return &bcashAddress{addr: addr}, nil - } - - da, err := address.NewFromString(addr) - if err != nil { - return nil, err - } - var ea string - switch format { - case CashAddr: - if a, err := da.CashAddress(); err != nil { - return nil, err - } else { - ea, err = a.Encode() - if err != nil { - return nil, err + // do not return unknown script type error as error + if err.Error() == "unknown script type" { + // try bitcoin parser to parse P2PK scripts - kind of hack as bchutil does not support pay-to-pubkey + _, addresses, _, err := txscript.ExtractPkScriptAddrs(script, p.Params) + if err == nil && len(addresses) == 1 { + // convert the address back to output script and back to get from bitcoin to bcash + s, err := p.addressToOutputScript(addresses[0].EncodeAddress()) + if err == nil { + a, err = bchutil.ExtractPkScriptAddrs(s, p.Params) + if err != nil { + return []string{}, false, nil + } + } + } else { + // try OP_RETURN script + or := btc.TryParseOPReturn(script) + if or != "" { + return []string{or}, false, nil + } + return []string{}, false, nil } - } - - case Legacy: - if a, err := da.Legacy(); err != nil { - return nil, err } else { - ea, err = a.Encode() - if err != nil { - return nil, err - } - } - default: - return nil, fmt.Errorf("Unknown address format: %d", format) - } - return &bcashAddress{addr: ea}, nil -} - -func (a *bcashAddress) String() string { - return a.addr -} - -func (a *bcashAddress) AreEqual(addr string) bool { - return a.String() == addr -} - -func (a *bcashAddress) InSlice(addrs []string) bool { - ea := a.String() - for _, addr := range addrs { - if ea == addr { - return true + return nil, false, err } } - return false + // EncodeAddress returns CashAddr address + addr := a.EncodeAddress() + if p.AddressFormat == Legacy { + da, err := address.NewFromString(addr) + if err != nil { + return nil, false, err + } + ca, err := da.Legacy() + if err != nil { + return nil, false, err + } + addr, err = ca.Encode() + if err != nil { + return nil, false, err + } + } + return []string{addr}, len(addr) > 0, nil } diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index 3e1cb4c1..71464b57 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -5,141 +5,218 @@ package bch import ( "blockbook/bchain" "blockbook/bchain/coins/btc" - "bytes" "encoding/hex" + "math/big" "reflect" "testing" ) -func TestBcashAddressEncodeAddress(t *testing.T) { - addr1, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", Legacy) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return +func Test_GetAddrDescFromAddress(t *testing.T) { + mainParserCashAddr, mainParserLegacy, testParserCashAddr, _ := setupParsers(t) + tests := []struct { + name string + parser *BCashParser + addresses []string + hex string + wantErr bool + }{ + { + name: "test-P2PKH-0", + parser: testParserCashAddr, + addresses: []string{"mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr"}, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "test-P2PKH-1", + parser: testParserCashAddr, + addresses: []string{"bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm"}, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserLegacy, + addresses: []string{"129HiRqekqPVucKy2M8zsqvafGgKypciPp"}, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqxgjelx8qk85t9xfk8g2zlunxmhxms6p55xarv2r5"}, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2SH-0", + parser: mainParserCashAddr, + addresses: []string{"3EBEFWPtDYWCNszQ7etoqtWmmygccayLiH"}, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, + { + name: "main-P2SH-1", + parser: mainParserLegacy, + addresses: []string{"bitcoincash:pzy0wuj9pjps5v8dmlwq32fatu4wrgcwzuayq5nfhh"}, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, } - got1 := addr1.String() - if got1 != "13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji" { - t.Errorf("String() got1 = %v, want %v", got1, "13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji") - } - addr2, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", CashAddr) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got2 := addr2.String() - if got2 != "bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf" { - t.Errorf("String() got2 = %v, want %v", got2, "bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.parser.GetAddrDescFromAddress(tt.addresses[0]) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.hex) { + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.hex) + } + }) } } -func TestBcashAddressAreEqual(t *testing.T) { - addr1, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", Legacy) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - addr2, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", CashAddr) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got1 := addr1.AreEqual("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji") - if got1 != true { - t.Errorf("AreEqual() got1 = %v, want %v", got1, true) - } - got2 := addr2.AreEqual("bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf") - if got2 != true { - t.Errorf("AreEqual() got2 = %v, want %v", got2, true) - } - got3 := addr1.AreEqual("1HoKgKQh7ZNomWURmS9Tk3z8JM2MWm7S1w") - if got3 != false { - t.Errorf("AreEqual() got3 = %v, want %v", got3, false) - } - got4 := addr2.AreEqual("bitcoincash:qzuyf0gpqj7q5wfck3nyghhklju7r0k3ksmq6d0vch") - if got4 != false { - t.Errorf("AreEqual() got4 = %v, want %v", got4, false) - } -} +func Test_GetAddressesFromAddrDesc(t *testing.T) { + mainParserCashAddr, mainParserLegacy, testParserCashAddr, testParserLegacy := setupParsers(t) + tests := []struct { + name string + parser *BCashParser + addresses []string + searchable bool + hex string + wantErr bool + }{ + { + name: "test-P2PKH-0", + parser: testParserLegacy, + addresses: []string{"mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr"}, + searchable: true, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", -func TestBcashAddressInSlice(t *testing.T) { - addr1, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", Legacy) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return + wantErr: false, + }, + { + name: "test-P2PKH-1", + parser: testParserCashAddr, + addresses: []string{"bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm"}, + searchable: true, + hex: "76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserLegacy, + addresses: []string{"129HiRqekqPVucKy2M8zsqvafGgKypciPp"}, + searchable: true, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2PKH-0", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqxgjelx8qk85t9xfk8g2zlunxmhxms6p55xarv2r5"}, + searchable: true, + hex: "76a9140c8967e6382c7a2ca64d8e850bfc99b7736e1a0d88ac", + wantErr: false, + }, + { + name: "main-P2SH-0", + parser: mainParserLegacy, + addresses: []string{"3EBEFWPtDYWCNszQ7etoqtWmmygccayLiH"}, + searchable: true, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, + { + name: "main-P2SH-1", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:pzy0wuj9pjps5v8dmlwq32fatu4wrgcwzuayq5nfhh"}, + searchable: true, + hex: "a91488f772450c830a30eddfdc08a93d5f2ae1a30e1787", + wantErr: false, + }, + { + name: "main-P2PK", + parser: mainParserCashAddr, + addresses: []string{"bitcoincash:qqr95pwp0w5jqnh9vcjl4qm4x45atr0er57n49pq75"}, + searchable: true, + hex: "2103db3c3977c5165058bf38c46f72d32f4e872112dbafc13083a948676165cd1603ac", + wantErr: false, + }, + { + name: "OP_RETURN ascii", + parser: mainParserCashAddr, + addresses: []string{"OP_RETURN (ahoj)"}, + searchable: false, + hex: "6a0461686f6a", + wantErr: false, + }, + { + name: "OP_RETURN hex", + parser: mainParserCashAddr, + addresses: []string{"OP_RETURN 2020f1686f6a20"}, + searchable: false, + hex: "6a072020f1686f6a20", + wantErr: false, + }, + { + name: "empty", + parser: mainParserCashAddr, + addresses: []string{}, + searchable: false, + hex: "", + wantErr: false, + }, } - addr2, err := newBCashAddress("13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", CashAddr) - if err != nil { - t.Errorf("newBCashAddress() error = %v", err) - return - } - got1 := addr1.InSlice([]string{"13zMwGC5bxRn9ckJ1mgxf7UR8qbbNe2iji", "1HoKgKQh7ZNomWURmS9Tk3z8JM2MWm7S1w"}) - if got1 != true { - t.Errorf("InSlice() got1 = %v, want %v", got1, true) - } - got2 := addr2.InSlice([]string{"bitcoincash:qzuyf0gpqj7q5wfck3nyghhklju7r0k3ksmq6d0vch", "bitcoincash:qqsvjuqqwgyzvz7zz9xcvxent0ul2xjs6y4d9qvsrf"}) - if got2 != true { - t.Errorf("InSlice() got2 = %v, want %v", got2, true) - } - got3 := addr1.InSlice([]string{"1HoKgKQh7ZNomWURmS9Tk3z8JM2MWm7S1w", "1E6Np6dUPYpBSdLMLuwBF8sRQ3cngdaRRY"}) - if got3 != false { - t.Errorf("InSlice() got3 = %v, want %v", got3, false) - } - got4 := addr2.InSlice([]string{"bitcoincash:qzuyf0gpqj7q5wfck3nyghhklju7r0k3ksmq6d0vch", "bitcoincash:qz8emmpenqgeg7et8xsz8prvhy6cqcalyyjcamt7e9"}) - if got4 != false { - t.Errorf("InSlice() got4 = %v, want %v", got4, false) - } -} -func TestAddressToOutputScript(t *testing.T) { - parser, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) - if err != nil { - t.Errorf("NewBCashParser() error = %v", err) - return - } - want, err := hex.DecodeString("76a9144fa927fd3bcf57d4e3c582c3d2eb2bd3df8df47c88ac") - if err != nil { - panic(err) - } - got1, err := parser.AddressToOutputScript("mnnAKPTSrWjgoi3uEYaQkHA1QEC5btFeBr") - if err != nil { - t.Errorf("AddressToOutputScript() error = %v", err) - return - } - if !bytes.Equal(got1, want) { - t.Errorf("AddressToOutputScript() got1 = %v, want %v", got1, want) - } - got2, err := parser.AddressToOutputScript("bchtest:qp86jfla8084048rckpv85ht90falr050s03ejaesm") - if err != nil { - t.Errorf("AddressToOutputScript() error = %v", err) - return - } - if !bytes.Equal(got2, want) { - t.Errorf("AddressToOutputScript() got2 = %v, want %v", got2, want) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.hex) + got, got2, err := tt.parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.addresses) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.addresses) + } + if !reflect.DeepEqual(got2, tt.searchable) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.searchable) + } + }) } } var ( testTx1, testTx2 bchain.Tx - - testTxPacked1 = "0001e2408ba8d7af5401000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700" - testTxPacked2 = "0007c91a899ab7da6a010000000001019d64f0c72a0d206001decbffaa722eb1044534c74eee7a5df8318e42a4323ec10000000017160014550da1f5d25a9dae2eafd6902b4194c4c6500af6ffffffff02809698000000000017a914cd668d781ece600efa4b2404dc91fd26b8b8aed8870553d7360000000017a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a8702473044022076aba4ad559616905fa51d4ddd357fc1fdb428d40cb388e042cdd1da4a1b7357022011916f90c712ead9a66d5f058252efd280439ad8956a967e95d437d246710bc9012102a80a5964c5612bb769ef73147b2cf3c149bc0fd4ecb02f8097629c94ab013ffd00000000" + testTxPacked1 = "0001e2408ba8d7af5401000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700" + testTxPacked2 = "0007c91a899ab7da6a010000000001019d64f0c72a0d206001decbffaa722eb1044534c74eee7a5df8318e42a4323ec10000000017160014550da1f5d25a9dae2eafd6902b4194c4c6500af6ffffffff02809698000000000017a914cd668d781ece600efa4b2404dc91fd26b8b8aed8870553d7360000000017a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a8702473044022076aba4ad559616905fa51d4ddd357fc1fdb428d40cb388e042cdd1da4a1b7357022011916f90c712ead9a66d5f058252efd280439ad8956a967e95d437d246710bc9012102a80a5964c5612bb769ef73147b2cf3c149bc0fd4ecb02f8097629c94ab013ffd00000000" ) -func init() { - var ( - addr1, addr2, addr3 bchain.Address - err error - ) - addr1, err = newBCashAddress("3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK", Legacy) - if err == nil { - addr2, err = newBCashAddress("2NByHN6A8QYkBATzxf4pRGbCSHD5CEN2TRu", Legacy) - } - if err == nil { - addr3, err = newBCashAddress("2MvZguYaGjM7JihBgNqgLF2Ca2Enb76Hj9D", Legacy) - } +func setupParsers(t *testing.T) (mainParserCashAddr, mainParserLegacy, testParserCashAddr, testParserLegacy *BCashParser) { + parser1, err := NewBCashParser(GetChainParams("main"), &btc.Configuration{AddressFormat: "cashaddr"}) if err != nil { - panic(err) + t.Fatalf("NewBCashParser() error = %v", err) } + parser2, err := NewBCashParser(GetChainParams("main"), &btc.Configuration{AddressFormat: "legacy"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + parser3, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "cashaddr"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + parser4, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) + if err != nil { + t.Fatalf("NewBCashParser() error = %v", err) + } + return parser1, parser2, parser3, parser4 +} + +func init() { testTx1 = bchain.Tx{ Hex: "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700", @@ -159,15 +236,14 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ "bitcoincash:pps5f4tu3tl5sjfvnhaeznsjpvst44eddugfcnqpy9", }, }, - Address: addr1, }, }, } @@ -190,42 +266,31 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ "bchtest:prxkdrtcrm8xqrh6fvjqfhy3l5nt3w9wmq9fmsvkmz", }, }, - Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ "bchtest:pqjxv4dah42v0erh6r4zxa0gdcxm9w8cpg0qw8tqf6", }, }, - Address: addr3, }, }, } } func Test_UnpackTx(t *testing.T) { - parser1, err := NewBCashParser(GetChainParams("main"), &btc.Configuration{AddressFormat: "legacy"}) - if err != nil { - t.Errorf("NewBCashParser() error = %v", err) - return - } - parser2, err := NewBCashParser(GetChainParams("test"), &btc.Configuration{AddressFormat: "legacy"}) - if err != nil { - t.Errorf("NewBCashParser() error = %v", err) - return - } + mainParser, _, testParser, _ := setupParsers(t) type args struct { packedTx string @@ -239,10 +304,10 @@ func Test_UnpackTx(t *testing.T) { wantErr bool }{ { - name: "btc-1", + name: "bcash-1", args: args{ packedTx: testTxPacked1, - parser: parser1, + parser: mainParser, }, want: &testTx1, want1: 123456, @@ -252,7 +317,7 @@ func Test_UnpackTx(t *testing.T) { name: "testnet-1", args: args{ packedTx: testTxPacked2, - parser: parser2, + parser: testParser, }, want: &testTx2, want1: 510234, diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 68af837a..ce374430 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -5,9 +5,10 @@ import ( "blockbook/bchain/coins/btc" "encoding/hex" "encoding/json" + "math/big" - "github.com/cpacia/bchutil" "github.com/golang/glog" + "github.com/jakm/bchutil" "github.com/juju/errors" ) @@ -100,7 +101,10 @@ func (b *BCashRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { if err != nil { return nil, errors.Annotatef(err, "hash %v", hash) } + // size is not returned by GetBlockHeader and would be overwritten + size := block.Size block.BlockHeader = *header + block.Size = size return block, nil } @@ -126,13 +130,35 @@ func (b *BCashRPC) GetBlockRaw(hash string) ([]byte, error) { return hex.DecodeString(res.Result) } +// GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids +func (b *BCashRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + glog.V(1).Info("rpc: getblock (verbosity=1) ", hash) + + res := btc.ResGetBlockInfo{} + req := cmdGetBlock{Method: "getblock"} + req.Params.BlockHash = hash + req.Params.Verbose = true + err := b.Call(&req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + if isErrBlockNotFound(res.Error) { + return nil, bchain.ErrBlockNotFound + } + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + return &res.Result, nil +} + // GetBlockFull returns block with given hash. func (b *BCashRPC) GetBlockFull(hash string) (*bchain.Block, error) { return nil, errors.New("Not implemented") } // EstimateSmartFee returns fee estimation. -func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := btc.ResEstimateSmartFee{} @@ -141,13 +167,18 @@ func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, err // conservative param is omitted err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } func isErrBlockNotFound(err *bchain.RPCError) bool { diff --git a/bchain/coins/bch/bcashrpc_test.go b/bchain/coins/bch/bcashrpc_test.go deleted file mode 100644 index 7056125b..00000000 --- a/bchain/coins/bch/bcashrpc_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// +build integration - -package bch - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewBCashRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*BCashRPC) - cli.Parser, err = NewBCashParser(GetChainParams(chain), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - - t, err := rpc.NewTest("Bcash", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Bcash Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestBCashRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestBCashRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestBCashRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestBCashRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -func TestBCashRPC_MempoolSync(t *testing.T) { - tests.mainnet.TestMempoolSync(t) -} - -func TestBCashRPC_GetMempoolEntry(t *testing.T) { - tests.mainnet.TestGetMempoolEntry(t) -} - -func TestBCashRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestBCashRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestBCashRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestBCashRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestBCashRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestBCashTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestBCashTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestBCashTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestBCashTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestBCashTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestBCashTestnetRPC_GetMempoolEntry(t *testing.T) { - tests.testnet.TestGetMempoolEntry(t) -} - -func TestBCashTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestBCashTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestBCashTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.testnet.TestGetBestBlockHash(t) -} - -func TestBCashTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.testnet.TestGetBestBlockHeight(t) -} - -func TestBCashTestnetRPC_GetBlockHeader(t *testing.T) { - tests.testnet.TestGetBlockHeader(t) -} diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 6469966e..35315d80 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math/big" "reflect" "time" @@ -26,29 +27,29 @@ import ( type blockChainFactory func(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) -var blockChainFactories = make(map[string]blockChainFactory) +var BlockChainFactories = make(map[string]blockChainFactory) func init() { - blockChainFactories["Bitcoin"] = btc.NewBitcoinRPC - blockChainFactories["Testnet"] = btc.NewBitcoinRPC - blockChainFactories["Zcash"] = zec.NewZCashRPC - blockChainFactories["Zcash Testnet"] = zec.NewZCashRPC - blockChainFactories["Ethereum"] = eth.NewEthereumRPC - blockChainFactories["Ethereum Classic"] = eth.NewEthereumRPC - blockChainFactories["Ethereum Testnet Ropsten"] = eth.NewEthereumRPC - blockChainFactories["Bcash"] = bch.NewBCashRPC - blockChainFactories["Bcash Testnet"] = bch.NewBCashRPC - blockChainFactories["Bgold"] = btg.NewBGoldRPC - blockChainFactories["Dash"] = dash.NewDashRPC - blockChainFactories["Dash Testnet"] = dash.NewDashRPC - blockChainFactories["Litecoin"] = litecoin.NewLitecoinRPC - blockChainFactories["Litecoin Testnet"] = litecoin.NewLitecoinRPC - blockChainFactories["Dogecoin"] = dogecoin.NewDogecoinRPC - blockChainFactories["Vertcoin"] = vertcoin.NewVertcoinRPC - blockChainFactories["Vertcoin Testnet"] = vertcoin.NewVertcoinRPC - blockChainFactories["Namecoin"] = namecoin.NewNamecoinRPC - blockChainFactories["Monacoin"] = monacoin.NewMonacoinRPC - blockChainFactories["Monacoin Testnet"] = monacoin.NewMonacoinRPC + BlockChainFactories["Bitcoin"] = btc.NewBitcoinRPC + BlockChainFactories["Testnet"] = btc.NewBitcoinRPC + BlockChainFactories["Zcash"] = zec.NewZCashRPC + BlockChainFactories["Zcash Testnet"] = zec.NewZCashRPC + BlockChainFactories["Ethereum"] = eth.NewEthereumRPC + BlockChainFactories["Ethereum Classic"] = eth.NewEthereumRPC + BlockChainFactories["Ethereum Testnet Ropsten"] = eth.NewEthereumRPC + BlockChainFactories["Bcash"] = bch.NewBCashRPC + BlockChainFactories["Bcash Testnet"] = bch.NewBCashRPC + BlockChainFactories["Bgold"] = btg.NewBGoldRPC + BlockChainFactories["Dash"] = dash.NewDashRPC + BlockChainFactories["Dash Testnet"] = dash.NewDashRPC + BlockChainFactories["Litecoin"] = litecoin.NewLitecoinRPC + BlockChainFactories["Litecoin Testnet"] = litecoin.NewLitecoinRPC + BlockChainFactories["Dogecoin"] = dogecoin.NewDogecoinRPC + BlockChainFactories["Vertcoin"] = vertcoin.NewVertcoinRPC + BlockChainFactories["Vertcoin Testnet"] = vertcoin.NewVertcoinRPC + BlockChainFactories["Namecoin"] = namecoin.NewNamecoinRPC + BlockChainFactories["Monacoin"] = monacoin.NewMonacoinRPC + BlockChainFactories["Monacoin Testnet"] = monacoin.NewMonacoinRPC } // GetCoinNameFromConfig gets coin name and coin shortcut from config file @@ -79,9 +80,9 @@ func NewBlockChain(coin string, configfile string, pushHandler func(bchain.Notif if err != nil { return nil, errors.Annotatef(err, "Error parsing file %v", configfile) } - bcf, ok := blockChainFactories[coin] + bcf, ok := BlockChainFactories[coin] if !ok { - return nil, errors.New(fmt.Sprint("Unsupported coin '", coin, "'. Must be one of ", reflect.ValueOf(blockChainFactories).MapKeys())) + return nil, errors.New(fmt.Sprint("Unsupported coin '", coin, "'. Must be one of ", reflect.ValueOf(BlockChainFactories).MapKeys())) } bc, err := bcf(config, pushHandler) if err != nil { @@ -131,9 +132,9 @@ func (c *blockChainWithMetrics) GetSubversion() string { return c.b.GetSubversion() } -func (c *blockChainWithMetrics) GetBlockChainInfo() (v string, err error) { - defer func(s time.Time) { c.observeRPCLatency("GetBlockChainInfo", s, err) }(time.Now()) - return c.b.GetBlockChainInfo() +func (c *blockChainWithMetrics) GetChainInfo() (v *bchain.ChainInfo, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetChainInfo", s, err) }(time.Now()) + return c.b.GetChainInfo() } func (c *blockChainWithMetrics) GetBestBlockHash() (v string, err error) { @@ -161,6 +162,11 @@ func (c *blockChainWithMetrics) GetBlock(hash string, height uint32) (v *bchain. return c.b.GetBlock(hash, height) } +func (c *blockChainWithMetrics) GetBlockInfo(hash string) (v *bchain.BlockInfo, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetBlockInfo", s, err) }(time.Now()) + return c.b.GetBlockInfo(hash) +} + func (c *blockChainWithMetrics) GetMempool() (v []string, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempool", s, err) }(time.Now()) return c.b.GetMempool() @@ -176,12 +182,12 @@ func (c *blockChainWithMetrics) GetTransactionForMempool(txid string) (v *bchain return c.b.GetTransactionForMempool(txid) } -func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateSmartFee(blocks int, conservative bool) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateSmartFee", s, err) }(time.Now()) return c.b.EstimateSmartFee(blocks, conservative) } -func (c *blockChainWithMetrics) EstimateFee(blocks int) (v float64, err error) { +func (c *blockChainWithMetrics) EstimateFee(blocks int) (v big.Int, err error) { defer func(s time.Time) { c.observeRPCLatency("EstimateFee", s, err) }(time.Now()) return c.b.EstimateFee(blocks) } @@ -205,6 +211,11 @@ func (c *blockChainWithMetrics) GetMempoolTransactions(address string) (v []stri return c.b.GetMempoolTransactions(address) } +func (c *blockChainWithMetrics) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []string, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactionsForAddrDesc", s, err) }(time.Now()) + return c.b.GetMempoolTransactionsForAddrDesc(addrDesc) +} + func (c *blockChainWithMetrics) GetMempoolEntry(txid string) (v *bchain.MempoolEntry, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempoolEntry", s, err) }(time.Now()) return c.b.GetMempoolEntry(txid) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 861c7f31..41f9f2ad 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -5,17 +5,18 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "math/big" vlq "github.com/bsm/go-vlq" "github.com/btcsuite/btcd/blockchain" - "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcutil" + "github.com/jakm/btcutil" + "github.com/jakm/btcutil/chaincfg" + "github.com/jakm/btcutil/txscript" ) // OutputScriptToAddressesFunc converts ScriptPubKey to bitcoin addresses -type OutputScriptToAddressesFunc func(script []byte, params *chaincfg.Params) ([]string, error) +type OutputScriptToAddressesFunc func(script []byte) ([]string, bool, error) // BitcoinParser handle type BitcoinParser struct { @@ -26,14 +27,15 @@ type BitcoinParser struct { // NewBitcoinParser returns new BitcoinParser instance func NewBitcoinParser(params *chaincfg.Params, c *Configuration) *BitcoinParser { - return &BitcoinParser{ - &bchain.BaseParser{ - AddressFactory: bchain.NewBaseAddress, + p := &BitcoinParser{ + BaseParser: &bchain.BaseParser{ BlockAddressesToKeep: c.BlockAddressesToKeep, + AmountDecimalPoint: 8, }, - params, - outputScriptToAddresses, + Params: params, } + p.OutputScriptToAddressesFunc = p.outputScriptToAddresses + return p } // GetChainParams contains network parameters for the main Bitcoin network, @@ -49,18 +51,28 @@ func GetChainParams(chain string) *chaincfg.Params { return &chaincfg.MainNetParams } -// GetAddrIDFromVout returns internal address representation of given transaction output -func (p *BitcoinParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { +// GetAddrDescFromVout returns internal address representation (descriptor) of given transaction output +func (p *BitcoinParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { return hex.DecodeString(output.ScriptPubKey.Hex) } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *BitcoinParser) GetAddrIDFromAddress(address string) ([]byte, error) { - return p.AddressToOutputScript(address) +// GetAddrDescFromAddress returns internal address representation (descriptor) of given address +func (p *BitcoinParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { + return p.addressToOutputScript(address) } -// AddressToOutputScript converts bitcoin address to ScriptPubKey -func (p *BitcoinParser) AddressToOutputScript(address string) ([]byte, error) { +// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable +func (p *BitcoinParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { + return p.OutputScriptToAddressesFunc(addrDesc) +} + +// GetScriptFromAddrDesc returns output script for given address descriptor +func (p *BitcoinParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]byte, error) { + return addrDesc, nil +} + +// addressToOutputScript converts bitcoin address to ScriptPubKey +func (p *BitcoinParser) addressToOutputScript(address string) ([]byte, error) { da, err := btcutil.DecodeAddress(address, p.Params) if err != nil { return nil, err @@ -72,22 +84,65 @@ func (p *BitcoinParser) AddressToOutputScript(address string) ([]byte, error) { return script, nil } -// OutputScriptToAddresses converts ScriptPubKey to bitcoin addresses -func (p *BitcoinParser) OutputScriptToAddresses(script []byte) ([]string, error) { - return p.OutputScriptToAddressesFunc(script, p.Params) +// TryParseOPReturn tries to process OP_RETURN script and return its string representation +func TryParseOPReturn(script []byte) string { + if len(script) > 1 && script[0] == txscript.OP_RETURN { + // trying 2 variants of OP_RETURN data + // 1) OP_RETURN OP_PUSHDATA1 + // 2) OP_RETURN + var data []byte + var l int + if script[1] == txscript.OP_PUSHDATA1 && len(script) > 2 { + l = int(script[2]) + data = script[3:] + if l != len(data) { + l = int(script[1]) + data = script[2:] + } + } else { + l = int(script[1]) + data = script[2:] + } + if l == len(data) { + isASCII := true + for _, c := range data { + if c < 32 || c > 127 { + isASCII = false + break + } + } + var ed string + if isASCII { + ed = "(" + string(data) + ")" + } else { + ed = hex.EncodeToString(data) + } + return "OP_RETURN " + ed + } + } + return "" } // outputScriptToAddresses converts ScriptPubKey to bitcoin addresses -func outputScriptToAddresses(script []byte, params *chaincfg.Params) ([]string, error) { - _, addresses, _, err := txscript.ExtractPkScriptAddrs(script, params) +func (p *BitcoinParser) outputScriptToAddresses(script []byte) ([]string, bool, error) { + sc, addresses, _, err := txscript.ExtractPkScriptAddrs(script, p.Params) if err != nil { - return nil, err + return nil, false, err } rv := make([]string, len(addresses)) for i, a := range addresses { rv[i] = a.EncodeAddress() } - return rv, nil + var s bool + if sc != txscript.NonStandardTy && sc != txscript.NullDataTy { + s = true + } else if len(rv) == 0 { + or := TryParseOPReturn(script) + if or != "" { + rv = []string{or} + } + } + return rv, s, nil } func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.Tx { @@ -115,7 +170,7 @@ func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.T for i, out := range t.TxOut { addrs := []string{} if parseAddresses { - addrs, _ = p.OutputScriptToAddresses(out.PkScript) + addrs, _, _ = p.OutputScriptToAddressesFunc(out.PkScript) } s := bchain.ScriptPubKey{ Hex: hex.EncodeToString(out.PkScript), @@ -123,8 +178,10 @@ func (p *BitcoinParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bchain.T // missing: Asm, // missing: Type, } + var vs big.Int + vs.SetInt64(out.Value) vout[i] = bchain.Vout{ - Value: float64(out.Value) / 1E8, + ValueSat: vs, N: uint32(i), ScriptPubKey: s, } @@ -152,17 +209,6 @@ func (p *BitcoinParser) ParseTx(b []byte) (*bchain.Tx, error) { } tx := p.TxFromMsgTx(&t, true) tx.Hex = hex.EncodeToString(b) - - for i, vout := range tx.Vout { - if len(vout.ScriptPubKey.Addresses) == 1 { - a, err := p.AddressFactory(vout.ScriptPubKey.Addresses[0]) - if err != nil { - return nil, err - } - tx.Vout[i].Address = a - } - } - return &tx, nil } @@ -180,7 +226,13 @@ func (p *BitcoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: w.Header.Timestamp.Unix(), + }, + Txs: txs, + }, nil } // PackTx packs transaction to byte array diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index d605f45b..80245180 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -5,11 +5,12 @@ package btc import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) -func TestAddressToOutputScript(t *testing.T) { +func Test_GetAddrDescFromAddress(t *testing.T) { type args struct { address string } @@ -48,20 +49,20 @@ func TestAddressToOutputScript(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -func TestOutputScriptToAddresses(t *testing.T) { +func Test_GetAddressesFromAddrDesc(t *testing.T) { type args struct { script string } @@ -69,43 +70,75 @@ func TestOutputScriptToAddresses(t *testing.T) { name string args args want []string + want2 bool wantErr bool }{ { name: "P2PKH", args: args{script: "76a914be027bf3eac907bd4ac8cb9c5293b6f37662722088ac"}, want: []string{"1JKgN43B9SyLuZH19H5ECvr4KcfrbVHzZ6"}, + want2: true, wantErr: false, }, { name: "P2SH", args: args{script: "a9140394b3cf9a44782c10105b93962daa8dba304d7f87"}, want: []string{"321x69Cb9HZLWwAWGiUBT1U81r1zPLnEjL"}, + want2: true, wantErr: false, }, { name: "P2WPKH", args: args{script: "00141c12afc6b2602607fdbc209f2a053c54ecd2c673"}, want: []string{"bc1qrsf2l34jvqnq0lduyz0j5pfu2nkd93nnq0qggn"}, + want2: true, wantErr: false, }, { name: "P2WSH", args: args{script: "002003973a40ec94c0d10f6f6f0e7a62ba2044b7d19db6ff2bf60651e17fb29d8d29"}, want: []string{"bc1qqwtn5s8vjnqdzrm0du885c46ypzt05vakmljhasx28shlv5a355sw5exgr"}, + want2: true, + wantErr: false, + }, + { + name: "OP_RETURN ascii", + args: args{script: "6a0461686f6a"}, + want: []string{"OP_RETURN (ahoj)"}, + want2: false, + wantErr: false, + }, + { + name: "OP_RETURN OP_PUSHDATA1 ascii", + args: args{script: "6a4c0b446c6f7568792074657874"}, + want: []string{"OP_RETURN (Dlouhy text)"}, + want2: false, + wantErr: false, + }, + { + name: "OP_RETURN hex", + args: args{script: "6a072020f1686f6a20"}, + want: []string{"OP_RETURN 2020f1686f6a20"}, + want2: false, wantErr: false, }, } + + parser := NewBitcoinParser(GetChainParams("main"), &Configuration{}) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b, _ := hex.DecodeString(tt.args.script) - got, err := outputScriptToAddresses(b, GetChainParams("main")) + got, got2, err := parser.GetAddressesFromAddrDesc(b) if (err != nil) != tt.wantErr { - t.Errorf("outputScriptToAddresses() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("outputScriptToAddresses() = %v, want %v", got, tt.want) + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) } }) } @@ -119,21 +152,6 @@ var ( ) func init() { - var ( - addr1, addr2, addr3 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK") - if err == nil { - addr2, err = bchain.NewBaseAddress("2NByHN6A8QYkBATzxf4pRGbCSHD5CEN2TRu") - } - if err == nil { - addr3, err = bchain.NewBaseAddress("2MvZguYaGjM7JihBgNqgLF2Ca2Enb76Hj9D") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "01000000017f9a22c9cbf54bd902400df746f138f37bcf5b4d93eb755820e974ba43ed5f42040000006a4730440220037f4ed5427cde81d55b9b6a2fd08c8a25090c2c2fff3a75c1a57625ca8a7118022076c702fe55969fa08137f71afd4851c48e31082dd3c40c919c92cdbc826758d30121029f6da5623c9f9b68a9baf9c1bc7511df88fa34c6c2f71f7c62f2f03ff48dca80feffffff019c9700000000000017a9146144d57c8aff48492c9dfb914e120b20bad72d6f8773d00700", Blocktime: 1519053802, @@ -152,15 +170,14 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.00038812, - N: 0, + ValueSat: *big.NewInt(38812), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a9146144d57c8aff48492c9dfb914e120b20bad72d6f87", Addresses: []string{ "3AZKvpKhSh1o8t1QrX3UeXG9d2BhCRnbcK", }, }, - Address: addr1, }, }, } @@ -183,26 +200,24 @@ func init() { }, Vout: []bchain.Vout{ { - Value: .1, - N: 0, + ValueSat: *big.NewInt(10000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914cd668d781ece600efa4b2404dc91fd26b8b8aed887", Addresses: []string{ "2NByHN6A8QYkBATzxf4pRGbCSHD5CEN2TRu", }, }, - Address: addr2, }, { - Value: 9.20081157, - N: 1, + ValueSat: *big.NewInt(920081157), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a914246655bdbd54c7e477d0ea2375e86e0db2b8f80a87", Addresses: []string{ "2MvZguYaGjM7JihBgNqgLF2Ca2Enb76Hj9D", }, }, - Address: addr3, }, }, } diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index e0211b28..27494586 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -8,6 +8,7 @@ import ( "encoding/json" "io" "io/ioutil" + "math/big" "net" "net/http" "time" @@ -36,19 +37,21 @@ type BitcoinRPC struct { } type Configuration struct { - CoinName string `json:"coin_name"` - CoinShortcut string `json:"coin_shortcut"` - RPCURL string `json:"rpc_url"` - RPCUser string `json:"rpc_user"` - RPCPass string `json:"rpc_pass"` - RPCTimeout int `json:"rpc_timeout"` - Parse bool `json:"parse"` - MessageQueueBinding string `json:"message_queue_binding"` - Subversion string `json:"subversion"` - BlockAddressesToKeep int `json:"block_addresses_to_keep"` - MempoolWorkers int `json:"mempool_workers"` - MempoolSubWorkers int `json:"mempool_sub_workers"` - AddressFormat string `json:"address_format"` + CoinName string `json:"coin_name"` + CoinShortcut string `json:"coin_shortcut"` + RPCURL string `json:"rpc_url"` + RPCUser string `json:"rpc_user"` + RPCPass string `json:"rpc_pass"` + RPCTimeout int `json:"rpc_timeout"` + Parse bool `json:"parse"` + MessageQueueBinding string `json:"message_queue_binding"` + Subversion string `json:"subversion"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` + MempoolWorkers int `json:"mempool_workers"` + MempoolSubWorkers int `json:"mempool_sub_workers"` + AddressFormat string `json:"address_format"` + SupportsEstimateFee bool `json:"supports_estimate_fee"` + SupportsEstimateSmartFee bool `json:"supports_estimate_smart_fee"` } // NewBitcoinRPC returns new BitcoinRPC instance. @@ -70,6 +73,9 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT if c.MempoolSubWorkers < 1 { c.MempoolSubWorkers = 1 } + // btc supports both calls, other coins overriding BitcoinRPC can change this + c.SupportsEstimateFee = true + c.SupportsEstimateSmartFee = true transport := &http.Transport{ Dial: (&net.Dialer{KeepAlive: 600 * time.Second}).Dial, @@ -96,10 +102,11 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT // and if successful it connects to ZeroMQ and creates mempool handler func (b *BitcoinRPC) GetChainInfoAndInitializeMempool(bc bchain.BlockChain) (string, error) { // try to connect to block chain and get some info - chainName, err := bc.GetBlockChainInfo() + ci, err := bc.GetChainInfo() if err != nil { return "", err } + chainName := ci.Chain mq, err := bchain.NewMQ(b.ChainConfig.MessageQueueBinding, b.pushHandler) if err != nil { @@ -211,10 +218,30 @@ type CmdGetBlockChainInfo struct { type ResGetBlockChainInfo struct { Error *bchain.RPCError `json:"error"` Result struct { - Chain string `json:"chain"` - Blocks int `json:"blocks"` - Headers int `json:"headers"` - Bestblockhash string `json:"bestblockhash"` + Chain string `json:"chain"` + Blocks int `json:"blocks"` + Headers int `json:"headers"` + Bestblockhash string `json:"bestblockhash"` + Difficulty json.Number `json:"difficulty"` + SizeOnDisk int64 `json:"size_on_disk"` + Warnings string `json:"warnings"` + } `json:"result"` +} + +// getnetworkinfo + +type CmdGetNetworkInfo struct { + Method string `json:"method"` +} + +type ResGetNetworkInfo struct { + Error *bchain.RPCError `json:"error"` + Result struct { + Version json.Number `json:"version"` + Subversion json.Number `json:"subversion"` + ProtocolVersion json.Number `json:"protocolversion"` + Timeoffset float64 `json:"timeoffset"` + Warnings string `json:"warnings"` } `json:"result"` } @@ -259,9 +286,14 @@ type ResGetBlockRaw struct { Result string `json:"result"` } +type BlockThin struct { + bchain.BlockHeader + Txids []string `json:"tx"` +} + type ResGetBlockThin struct { Error *bchain.RPCError `json:"error"` - Result bchain.ThinBlock `json:"result"` + Result BlockThin `json:"result"` } type ResGetBlockFull struct { @@ -269,6 +301,11 @@ type ResGetBlockFull struct { Result bchain.Block `json:"result"` } +type ResGetBlockInfo struct { + Error *bchain.RPCError `json:"error"` + Result bchain.BlockInfo `json:"result"` +} + // getrawtransaction type CmdGetRawTransaction struct { @@ -302,8 +339,8 @@ type CmdEstimateSmartFee struct { type ResEstimateSmartFee struct { Error *bchain.RPCError `json:"error"` Result struct { - Feerate float64 `json:"feerate"` - Blocks int `json:"blocks"` + Feerate json.Number `json:"feerate"` + Blocks int `json:"blocks"` } `json:"result"` } @@ -318,7 +355,7 @@ type CmdEstimateFee struct { type ResEstimateFee struct { Error *bchain.RPCError `json:"error"` - Result float64 `json:"result"` + Result json.Number `json:"result"` } // sendrawtransaction @@ -380,21 +417,48 @@ func (b *BitcoinRPC) GetBestBlockHeight() (uint32, error) { return res.Result, nil } -// GetBlockChainInfo returns the name of the block chain: main/test/regtest. -func (b *BitcoinRPC) GetBlockChainInfo() (string, error) { +// GetChainInfo returns information about the connected backend +func (b *BitcoinRPC) GetChainInfo() (*bchain.ChainInfo, error) { glog.V(1).Info("rpc: getblockchaininfo") - res := ResGetBlockChainInfo{} - req := CmdGetBlockChainInfo{Method: "getblockchaininfo"} - err := b.Call(&req, &res) - + resCi := ResGetBlockChainInfo{} + err := b.Call(&CmdGetBlockChainInfo{Method: "getblockchaininfo"}, &resCi) if err != nil { - return "", err + return nil, err } - if res.Error != nil { - return "", res.Error + if resCi.Error != nil { + return nil, resCi.Error } - return res.Result.Chain, nil + + glog.V(1).Info("rpc: getnetworkinfo") + resNi := ResGetNetworkInfo{} + err = b.Call(&CmdGetNetworkInfo{Method: "getnetworkinfo"}, &resNi) + if err != nil { + return nil, err + } + if resNi.Error != nil { + return nil, resNi.Error + } + + rv := &bchain.ChainInfo{ + Bestblockhash: resCi.Result.Bestblockhash, + Blocks: resCi.Result.Blocks, + Chain: resCi.Result.Chain, + Difficulty: string(resCi.Result.Difficulty), + Headers: resCi.Result.Headers, + SizeOnDisk: resCi.Result.SizeOnDisk, + Subversion: string(resNi.Result.Subversion), + Timeoffset: resNi.Result.Timeoffset, + } + rv.Version = string(resNi.Result.Version) + rv.ProtocolVersion = string(resNi.Result.ProtocolVersion) + if len(resCi.Result.Warnings) > 0 { + rv.Warnings = resCi.Result.Warnings + " " + } + if resCi.Result.Warnings != resNi.Result.Warnings { + rv.Warnings += resNi.Result.Warnings + } + return rv, nil } func isErrBlockNotFound(err *bchain.RPCError) bool { @@ -448,7 +512,7 @@ func (b *BitcoinRPC) GetBlockHeader(hash string) (*bchain.BlockHeader, error) { // GetBlock returns block with given hash. func (b *BitcoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { var err error - if hash == "" && height > 0 { + if hash == "" { hash, err = b.GetBlockHash(height) if err != nil { return nil, err @@ -477,7 +541,29 @@ func (b *BitcoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) return block, nil } -// getBlockWithoutHeader is an optimization - it does not call GetBlockHeader to get prev, next hashes +// GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids +func (b *BitcoinRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + glog.V(1).Info("rpc: getblock (verbosity=1) ", hash) + + res := ResGetBlockInfo{} + req := CmdGetBlock{Method: "getblock"} + req.Params.BlockHash = hash + req.Params.Verbosity = 1 + err := b.Call(&req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + if isErrBlockNotFound(res.Error) { + return nil, bchain.ErrBlockNotFound + } + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + return &res.Result, nil +} + +// GetBlockWithoutHeader is an optimization - it does not call GetBlockHeader to get prev, next hashes // instead it sets to header only block hash and height passed in parameters func (b *BitcoinRPC) GetBlockWithoutHeader(hash string, height uint32) (*bchain.Block, error) { data, err := b.GetBlockRaw(hash) @@ -611,13 +697,23 @@ func (b *BitcoinRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, err return b.Mempool.Resync(onNewTxAddr) } -// GetMempoolTransactions returns slice of mempool transactions for given address. +// GetMempoolTransactions returns slice of mempool transactions for given address func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]string, error) { return b.Mempool.GetTransactions(address) } -// EstimateSmartFee returns fee estimation. -func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +// GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor +func (b *BitcoinRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, error) { + return b.Mempool.GetAddrDescTransactions(addrDesc) +} + +// EstimateSmartFee returns fee estimation +func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + // use EstimateFee if EstimateSmartFee is not supported + if !b.ChainConfig.SupportsEstimateSmartFee && b.ChainConfig.SupportsEstimateFee { + return b.EstimateFee(blocks) + } + glog.V(1).Info("rpc: estimatesmartfee ", blocks) res := ResEstimateSmartFee{} @@ -630,17 +726,27 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, e } err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result.Feerate, nil + r, err = b.Parser.AmountToBigInt(res.Result.Feerate) + if err != nil { + return r, err + } + return r, nil } // EstimateFee returns fee estimation. -func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { +func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { + // use EstimateSmartFee if EstimateFee is not supported + if !b.ChainConfig.SupportsEstimateFee && b.ChainConfig.SupportsEstimateSmartFee { + return b.EstimateSmartFee(blocks, true) + } + glog.V(1).Info("rpc: estimatefee ", blocks) res := ResEstimateFee{} @@ -648,13 +754,18 @@ func (b *BitcoinRPC) EstimateFee(blocks int) (float64, error) { req.Params.Blocks = blocks err := b.Call(&req, &res) + var r big.Int if err != nil { - return 0, err + return r, err } if res.Error != nil { - return 0, res.Error + return r, res.Error } - return res.Result, nil + r, err = b.Parser.AmountToBigInt(res.Result) + if err != nil { + return r, err + } + return r, nil } // SendRawTransaction sends raw transaction. @@ -685,13 +796,20 @@ func (b *BitcoinRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) Params: []string{txid}, } err := b.Call(&req, &res) - if err != nil { return nil, err } if res.Error != nil { return nil, res.Error } + res.Result.FeeSat, err = b.Parser.AmountToBigInt(res.Result.Fee) + if err != nil { + return nil, err + } + res.Result.ModifiedFeeSat, err = b.Parser.AmountToBigInt(res.Result.ModifiedFee) + if err != nil { + return nil, err + } return res.Result, nil } diff --git a/bchain/coins/btc/bitcoinrpc_test.go b/bchain/coins/btc/bitcoinrpc_test.go deleted file mode 100644 index 882a1bd7..00000000 --- a/bchain/coins/btc/bitcoinrpc_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// +build integration - -package btc - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewBitcoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*BitcoinRPC) - cli.Parser = NewBitcoinParser(GetChainParams(chain), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Bitcoin", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Bitcoin Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestBitcoinRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestBitcoinRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestBitcoinRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestBitcoinRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -// FIXME -func TestBitcoinRPC_MempoolSync(t *testing.T) { - t.Skip("skipping test, run too long") - // tests.mainnet.TestMempoolSync(t) -} - -func TestBitcoinRPC_GetMempoolEntry(t *testing.T) { - tests.mainnet.TestGetMempoolEntry(t) -} - -func TestBitcoinRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestBitcoinRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestBitcoinRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestBitcoinRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestBitcoinRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestBitcoinTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestBitcoinTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestBitcoinTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestBitcoinTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestBitcoinTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestBitcoinTestnetRPC_GetMempoolEntry(t *testing.T) { - tests.testnet.TestGetMempoolEntry(t) -} - -func TestBitcoinTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestBitcoinTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestBitcoinTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.testnet.TestGetBestBlockHash(t) -} - -func TestBitcoinTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.testnet.TestGetBestBlockHeight(t) -} - -func TestBitcoinTestnetRPC_GetBlockHeader(t *testing.T) { - tests.testnet.TestGetBlockHeader(t) -} diff --git a/bchain/coins/btg/bgoldparser.go b/bchain/coins/btg/bgoldparser.go index 67d2c7ef..1e87aa93 100644 --- a/bchain/coins/btg/bgoldparser.go +++ b/bchain/coins/btg/bgoldparser.go @@ -5,11 +5,12 @@ import ( "blockbook/bchain/coins/btc" "blockbook/bchain/coins/utils" "bytes" + "encoding/binary" "io" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -22,13 +23,13 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic // Address encoding magics - MainNetParams.PubKeyHashAddrID = 38 // base58 prefix: G - MainNetParams.ScriptHashAddrID = 23 // base58 prefix: A + MainNetParams.PubKeyHashAddrID = []byte{38} // base58 prefix: G + MainNetParams.ScriptHashAddrID = []byte{23} // base58 prefix: A TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic @@ -62,6 +63,9 @@ func NewBGoldParser(params *chaincfg.Params, c *btc.Configuration) *BGoldParser // the regression test Bitcoin Cash network, the test Bitcoin Cash network and // the simulation test Bitcoin Cash network, in this order func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams @@ -75,11 +79,13 @@ func GetChainParams(chain string) *chaincfg.Params { // headerFixedLength is the length of fixed fields of a block (i.e. without solution) // see https://github.com/BTCGPU/BTCGPU/wiki/Technical-Spec#block-header const headerFixedLength = 44 + (chainhash.HashSize * 3) +const timestampOffset = 100 +const timestampLength = 4 // ParseBlock parses raw block to our Block struct func (p *BGoldParser) ParseBlock(b []byte) (*bchain.Block, error) { r := bytes.NewReader(b) - err := skipHeader(r, 0) + time, err := getTimestampAndSkipHeader(r, 0) if err != nil { return nil, err } @@ -95,24 +101,41 @@ func (p *BGoldParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: time, + }, + Txs: txs, + }, nil } -func skipHeader(r io.ReadSeeker, pver uint32) error { - _, err := r.Seek(headerFixedLength, io.SeekStart) +func getTimestampAndSkipHeader(r io.ReadSeeker, pver uint32) (int64, error) { + _, err := r.Seek(timestampOffset, io.SeekStart) if err != nil { - return err + return 0, err + } + + buf := make([]byte, timestampLength) + if _, err = io.ReadFull(r, buf); err != nil { + return 0, err + } + time := binary.LittleEndian.Uint32(buf) + + _, err = r.Seek(headerFixedLength-timestampOffset-timestampLength, io.SeekCurrent) + if err != nil { + return 0, err } size, err := wire.ReadVarInt(r, pver) if err != nil { - return err + return 0, err } _, err = r.Seek(int64(size), io.SeekCurrent) if err != nil { - return err + return 0, err } - return nil + return int64(time), nil } diff --git a/bchain/coins/btg/bgoldparser_test.go b/bchain/coins/btg/bgoldparser_test.go index 8834cfc0..e37fecd7 100644 --- a/bchain/coins/btg/bgoldparser_test.go +++ b/bchain/coins/btg/bgoldparser_test.go @@ -12,72 +12,86 @@ import ( "testing" ) -var testParseBlockTxs = map[int][]string{ - 104000: []string{ - "331d4ef64118e9e5be75f0f51f1a4c5057550c3320e22ff7206f3e1101f113d0", - "1f4817d8e91c21d8c8d163dabccdd1875f760fd2dc34a1c2b7b8fa204e103597", - "268163b1a1092aa0996d118a6027b0b6f1076627e02addc4e66ae30239936818", - "27277a1049fafa2a46368ad02961d37da633416b014bcd42a1f1391753cbf559", - "276d2d331585d0968762d9788f71ae71524ccba3494f638b2328ac51e52edd3d", - "28d9f85089834c20507cc5e4ec54aaaf5b79feab80cad24a48b8296b6d327a43", - "2a2d66d3d9e8b6f154958a388377f681abd82ce98785a5bbf2e27d0ca454da3f", - "39c9e995a12b638b541d21ed9f9e555716709add6a97c8ba63fe26e4d26bdc3f", - "4a768efc6cc0716932800115989376c2ce3e5e17668b08bd2f43031105a6ac6e", - "4bc41fa0188d988d853a176eb847a541c5adf35348eab21d128904aab36e8574", - "4cad1bd38cc7be880f3a968af6f13a3aeb5dbdb51a774b7d1ae3d6d6bfd114e4", - "6bc558801583bfdb656106889c4b8bd783168133784666338c57e5b2a675a922", - "75eb5c1aa89b18ce71f363f95147942b46645ca9b1e472784fcb8c9a096f4f5c", - "91755365cff22c4eed09a57a8fb7b2faa5c1caa5fa750c89f830a2bb56fa4c68", - "9417d34969891f2a0b9aa3a1226505edf3b429fa1acd21a140d358a217d11d55", - "950fbb5f413af9f7c6e5dabfacf68b715c9851b5cf6ab6806960f7cc7cad2f9d", - "99b134dae55ddfab1f5d5243c2e60030a9ed969ba5915f98840b877f8af28ce0", - "9d7b15eaaccce66e25efe7e2979454ae4968b61281d50f32be9872d2d256c244", - "a54df5296f1d1a6101cee0869ffda03502e417fc72475b7902a6dfd5b9329399", - "adba400f14b476f0c2b11340ee1fa440208b49fd99c1072136198b5c43664826", - "bd7d8fee068bd45b06b4c17ccdf577b4bc2b21c9c4e0cee8453409b0e63b4f5d", - "beabd2d68b66a9b47b6aff23b569c1b59e429074f06bdd4993e9d3c2cb69c992", - "bfa81174d549eb7ed15be3f965686aff3084f22523c52fbed03c3fc3e18b7cda", - "e42472099cb039b1c2248b611ce212580338550977e02bd77accdf29bfd86e96", - "f056e02b12d99377f724a4987cde68ecf6f234fd7e2bdf4324172c03d015ba25", - "f1815cfb1ef4cfe13ff5ec2c15b5bc55fde043db85daca1bb34cc1b491193308", - "f225abce64f75383686fa08abe47242e59e97809a31c8fd7a88acce1e6cbcd27", - "f93f1b125bfa2da5ccaaf30ff96635b905b657d48a5962c24be93884a82ef354", - "fef75d015f2e9926d1d4bf82e567b91e51af66a6e636d03a072f203dd3062ae5", - "051b60a6accead85da54b8d18f4b2360ea946da948d3a27836306d2927fed13e", - "28e47b050ec4064cdbd3364f3be9445d52635e9730691ef71ed1db0f0147afd6", - "446ebde2457102bcbc2c86cac6ff582c595b00346fd0b27ea5a645240020504b", - "46c8fafe2b7bb1646aeefa229b18fa7ffe20fee0a30c4a9ef3e63c36c808f6f7", - "550d96cf82fbe91dcc9b96e7aa404f392ee47400c22a98a7631d29eee43fceaa", - "59b6b78a72cc33cd29741894b3007b1330fc7f7945bdc0a7a4044ed1dd289c19", - "5a3aa07474338cf193c1d7aacdc37f3311c971857ba8cfd308e8fabf5e473882", - "82e014b1a9c6cb7729274653ce748c66953de6abb3d1411471515b41b727cf75", - "8d70af4f135696da396c9aa9f936b54195bfbe6ff0e08b3b210ca0b52bc167d2", - "9949c2f2f3b96a557ef6e14004cbd239a0744c056faca34bdff01e125b4380e8", - "d09a8c83123ce1cb6ff837e7670aab5f50c5155d9706dd26f7e0761fd20c5536", - "f601482efc5b2dd3d0031e318e840cd06f7cab0ffff8cc37a5bf471b515ddfb7", - "f88d3c0ebe8b294f11e70d2ae6d2f0048437bfb20dae7e69d545a4c72d3432dd", - "2b9e574b90556250b20a79d9c94ceaff3dfd062291c34c3fa79c7ca8d85a3500", - "b9484ef8e38ceafe8d2b09ecf59562d262b15d185844b2d00db362718d52b2c2", - "07a4af0a81b55313a1c16da7d808829d689436fd078fa9559b6d1603dd72474e", - "3393bdcc3a7232b37d0fb6b56d603a2b9b0419e461bf989f1c375859a5d0156a", - "33ad36d79d63b575c7532c516f16b19541f5c637caf7073beb7ddf604c3f39cc", +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ + 104000: testBlock{ + size: 15776, + time: 1295705889, + txs: []string{ + "331d4ef64118e9e5be75f0f51f1a4c5057550c3320e22ff7206f3e1101f113d0", + "1f4817d8e91c21d8c8d163dabccdd1875f760fd2dc34a1c2b7b8fa204e103597", + "268163b1a1092aa0996d118a6027b0b6f1076627e02addc4e66ae30239936818", + "27277a1049fafa2a46368ad02961d37da633416b014bcd42a1f1391753cbf559", + "276d2d331585d0968762d9788f71ae71524ccba3494f638b2328ac51e52edd3d", + "28d9f85089834c20507cc5e4ec54aaaf5b79feab80cad24a48b8296b6d327a43", + "2a2d66d3d9e8b6f154958a388377f681abd82ce98785a5bbf2e27d0ca454da3f", + "39c9e995a12b638b541d21ed9f9e555716709add6a97c8ba63fe26e4d26bdc3f", + "4a768efc6cc0716932800115989376c2ce3e5e17668b08bd2f43031105a6ac6e", + "4bc41fa0188d988d853a176eb847a541c5adf35348eab21d128904aab36e8574", + "4cad1bd38cc7be880f3a968af6f13a3aeb5dbdb51a774b7d1ae3d6d6bfd114e4", + "6bc558801583bfdb656106889c4b8bd783168133784666338c57e5b2a675a922", + "75eb5c1aa89b18ce71f363f95147942b46645ca9b1e472784fcb8c9a096f4f5c", + "91755365cff22c4eed09a57a8fb7b2faa5c1caa5fa750c89f830a2bb56fa4c68", + "9417d34969891f2a0b9aa3a1226505edf3b429fa1acd21a140d358a217d11d55", + "950fbb5f413af9f7c6e5dabfacf68b715c9851b5cf6ab6806960f7cc7cad2f9d", + "99b134dae55ddfab1f5d5243c2e60030a9ed969ba5915f98840b877f8af28ce0", + "9d7b15eaaccce66e25efe7e2979454ae4968b61281d50f32be9872d2d256c244", + "a54df5296f1d1a6101cee0869ffda03502e417fc72475b7902a6dfd5b9329399", + "adba400f14b476f0c2b11340ee1fa440208b49fd99c1072136198b5c43664826", + "bd7d8fee068bd45b06b4c17ccdf577b4bc2b21c9c4e0cee8453409b0e63b4f5d", + "beabd2d68b66a9b47b6aff23b569c1b59e429074f06bdd4993e9d3c2cb69c992", + "bfa81174d549eb7ed15be3f965686aff3084f22523c52fbed03c3fc3e18b7cda", + "e42472099cb039b1c2248b611ce212580338550977e02bd77accdf29bfd86e96", + "f056e02b12d99377f724a4987cde68ecf6f234fd7e2bdf4324172c03d015ba25", + "f1815cfb1ef4cfe13ff5ec2c15b5bc55fde043db85daca1bb34cc1b491193308", + "f225abce64f75383686fa08abe47242e59e97809a31c8fd7a88acce1e6cbcd27", + "f93f1b125bfa2da5ccaaf30ff96635b905b657d48a5962c24be93884a82ef354", + "fef75d015f2e9926d1d4bf82e567b91e51af66a6e636d03a072f203dd3062ae5", + "051b60a6accead85da54b8d18f4b2360ea946da948d3a27836306d2927fed13e", + "28e47b050ec4064cdbd3364f3be9445d52635e9730691ef71ed1db0f0147afd6", + "446ebde2457102bcbc2c86cac6ff582c595b00346fd0b27ea5a645240020504b", + "46c8fafe2b7bb1646aeefa229b18fa7ffe20fee0a30c4a9ef3e63c36c808f6f7", + "550d96cf82fbe91dcc9b96e7aa404f392ee47400c22a98a7631d29eee43fceaa", + "59b6b78a72cc33cd29741894b3007b1330fc7f7945bdc0a7a4044ed1dd289c19", + "5a3aa07474338cf193c1d7aacdc37f3311c971857ba8cfd308e8fabf5e473882", + "82e014b1a9c6cb7729274653ce748c66953de6abb3d1411471515b41b727cf75", + "8d70af4f135696da396c9aa9f936b54195bfbe6ff0e08b3b210ca0b52bc167d2", + "9949c2f2f3b96a557ef6e14004cbd239a0744c056faca34bdff01e125b4380e8", + "d09a8c83123ce1cb6ff837e7670aab5f50c5155d9706dd26f7e0761fd20c5536", + "f601482efc5b2dd3d0031e318e840cd06f7cab0ffff8cc37a5bf471b515ddfb7", + "f88d3c0ebe8b294f11e70d2ae6d2f0048437bfb20dae7e69d545a4c72d3432dd", + "2b9e574b90556250b20a79d9c94ceaff3dfd062291c34c3fa79c7ca8d85a3500", + "b9484ef8e38ceafe8d2b09ecf59562d262b15d185844b2d00db362718d52b2c2", + "07a4af0a81b55313a1c16da7d808829d689436fd078fa9559b6d1603dd72474e", + "3393bdcc3a7232b37d0fb6b56d603a2b9b0419e461bf989f1c375859a5d0156a", + "33ad36d79d63b575c7532c516f16b19541f5c637caf7073beb7ddf604c3f39cc", + }, }, - 532144: []string{ - "574348e23301cc89535408b6927bf75f2ac88fadf8fdfb181c17941a5de02fe0", - "9f048446401e7fac84963964df045b1f3992eda330a87b02871e422ff0a3fd28", - "9516c320745a227edb07c98087b1febea01c3ba85122a34387896fc82ba965e4", - "9d37e1ab5a28c49ce5e7ece4a2b9df740fb4c3a84bdec93b3023148cf20f0de7", - "a3cd0481b983ba402fed8805ef9daf5063d6d4e5085b82eca5b4781c9e362d6a", - "7f2c2567e8de0321744817cfeb751922d7e8d82ef71aa01164c84fb74a463a53", - "cd064315e3f5d07920b3d159160c218d0bb5b7b4be606265767b208ae7e256eb", - "a9523400f341aa425b0fcc00656ec1fa5421bf3545433bff98a8614fc9a99d1f", - "ec766daacbb05a8f48a3205e5c6494a7c817bd35eefff9aaf55e0dd47fe6e8fc", - "0837a4116872abf52caa52d1ff7608674ba5b09a239a1f43f3a25ba4052e4c77", - "a3e23a0344fe6ba7083fc6afb940517cdb666dce00389cbd8598bd599199cdda", - "048d951cef84d35d68f0bc3b575662caf23fee692e8035bd5efe38ab67e0d6c2", - "11307491b24d42ddd7ea27fc795d444b65c3936be31b904a97da68fabc85e5b8", - "84ad99dc0884e03fc71f163eebf515a1eb79d00b1aad7a1126b22747960a8275", - "728c8d0858e506d4a1a9b506f7b974b335e6c54047af9f40d4cb1a0561f783e1", + 532144: testBlock{ + size: 12198, + time: 1528372417, + txs: []string{ + "574348e23301cc89535408b6927bf75f2ac88fadf8fdfb181c17941a5de02fe0", + "9f048446401e7fac84963964df045b1f3992eda330a87b02871e422ff0a3fd28", + "9516c320745a227edb07c98087b1febea01c3ba85122a34387896fc82ba965e4", + "9d37e1ab5a28c49ce5e7ece4a2b9df740fb4c3a84bdec93b3023148cf20f0de7", + "a3cd0481b983ba402fed8805ef9daf5063d6d4e5085b82eca5b4781c9e362d6a", + "7f2c2567e8de0321744817cfeb751922d7e8d82ef71aa01164c84fb74a463a53", + "cd064315e3f5d07920b3d159160c218d0bb5b7b4be606265767b208ae7e256eb", + "a9523400f341aa425b0fcc00656ec1fa5421bf3545433bff98a8614fc9a99d1f", + "ec766daacbb05a8f48a3205e5c6494a7c817bd35eefff9aaf55e0dd47fe6e8fc", + "0837a4116872abf52caa52d1ff7608674ba5b09a239a1f43f3a25ba4052e4c77", + "a3e23a0344fe6ba7083fc6afb940517cdb666dce00389cbd8598bd599199cdda", + "048d951cef84d35d68f0bc3b575662caf23fee692e8035bd5efe38ab67e0d6c2", + "11307491b24d42ddd7ea27fc795d444b65c3936be31b904a97da68fabc85e5b8", + "84ad99dc0884e03fc71f163eebf515a1eb79d00b1aad7a1126b22747960a8275", + "728c8d0858e506d4a1a9b506f7b974b335e6c54047af9f40d4cb1a0561f783e1", + }, }, } @@ -104,7 +118,7 @@ func helperLoadBlock(t *testing.T, height int) []byte { func TestParseBlock(t *testing.T) { p := NewBGoldParser(GetChainParams("main"), &btc.Configuration{}) - for height, txs := range testParseBlockTxs { + for height, tb := range testParseBlockTxs { b := helperLoadBlock(t, height) blk, err := p.ParseBlock(b) @@ -112,11 +126,19 @@ func TestParseBlock(t *testing.T) { t.Fatal(err) } - if len(blk.Txs) != len(txs) { - t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(txs)) + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) } - for ti, tx := range txs { + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { if blk.Txs[ti].Txid != tx { t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) } diff --git a/bchain/coins/dash/dashparser.go b/bchain/coins/dash/dashparser.go index 5590cc87..d2371ef5 100644 --- a/bchain/coins/dash/dashparser.go +++ b/bchain/coins/dash/dashparser.go @@ -3,8 +3,8 @@ package dash import ( "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -19,27 +19,27 @@ var ( RegtestParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic // Address encoding magics - MainNetParams.PubKeyHashAddrID = 76 // base58 prefix: X - MainNetParams.ScriptHashAddrID = 16 // base58 prefix: 7 + MainNetParams.PubKeyHashAddrID = []byte{76} // base58 prefix: X + MainNetParams.ScriptHashAddrID = []byte{16} // base58 prefix: 7 TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic // Address encoding magics - TestNetParams.PubKeyHashAddrID = 140 // base58 prefix: y - TestNetParams.ScriptHashAddrID = 19 // base58 prefix: 8 or 9 + TestNetParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + TestNetParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 RegtestParams = chaincfg.RegressionNetParams RegtestParams.Net = RegtestMagic // Address encoding magics - RegtestParams.PubKeyHashAddrID = 140 // base58 prefix: y - RegtestParams.ScriptHashAddrID = 19 // base58 prefix: 8 or 9 + RegtestParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + RegtestParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 err := chaincfg.Register(&MainNetParams) if err == nil { @@ -67,6 +67,9 @@ func NewDashParser(params *chaincfg.Params, c *btc.Configuration) *DashParser { // the regression test Dash network, the test Dash network and // the simulation test Dash network, in this order func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/dash/dashrpc.go b/bchain/coins/dash/dashrpc.go index 4c884f08..dec0ee49 100644 --- a/bchain/coins/dash/dashrpc.go +++ b/bchain/coins/dash/dashrpc.go @@ -24,6 +24,7 @@ func NewDashRPC(config json.RawMessage, pushHandler func(bchain.NotificationType b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -53,8 +54,3 @@ func (b *DashRPC) Initialize() error { return nil } - -// EstimateSmartFee returns fee estimation. -func (b *DashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/dash/dashrpc_test.go b/bchain/coins/dash/dashrpc_test.go deleted file mode 100644 index 98662672..00000000 --- a/bchain/coins/dash/dashrpc_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// +build integration - -package dash - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(chain string) func(json.RawMessage) (bchain.BlockChain, error) { - return func(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewDashRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*DashRPC) - cli.Parser = NewDashParser(GetChainParams(chain), cli.ChainConfig) - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil - } -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Dash", getRPCClient("main")) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Dash Testnet", getRPCClient("test")) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestDashRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestDashRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestDashRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestDashRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -func TestDashRPC_MempoolSync(t *testing.T) { - tests.mainnet.TestMempoolSync(t) -} - -func TestDashRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestDashRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestDashRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestDashRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestDashRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestDashTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestDashTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestDashTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestDashTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestDashTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestDashTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestDashTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestDashTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.testnet.TestGetBestBlockHash(t) -} - -func TestDashTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.testnet.TestGetBestBlockHeight(t) -} - -func TestDashTestnetRPC_GetBlockHeader(t *testing.T) { - tests.testnet.TestGetBlockHeader(t) -} diff --git a/bchain/coins/dogecoin/dogecoinparser.go b/bchain/coins/dogecoin/dogecoinparser.go index ac7ea94b..56c95983 100644 --- a/bchain/coins/dogecoin/dogecoinparser.go +++ b/bchain/coins/dogecoin/dogecoinparser.go @@ -6,8 +6,8 @@ import ( "blockbook/bchain/coins/utils" "bytes" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -18,11 +18,11 @@ var ( MainNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 30 - MainNetParams.ScriptHashAddrID = 22 + MainNetParams.PubKeyHashAddrID = []byte{30} + MainNetParams.ScriptHashAddrID = []byte{22} err := chaincfg.Register(&MainNetParams) if err != nil { @@ -43,6 +43,9 @@ func NewDogecoinParser(params *chaincfg.Params, c *btc.Configuration) *DogecoinP // GetChainParams contains network parameters for the main Dogecoin network, // and the test Dogecoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { default: return &MainNetParams @@ -75,5 +78,11 @@ func (p *DogecoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: h.Timestamp.Unix(), + }, + Txs: txs, + }, nil } diff --git a/bchain/coins/dogecoin/dogecoinparser_test.go b/bchain/coins/dogecoin/dogecoinparser_test.go index b8ea585f..5638f664 100644 --- a/bchain/coins/dogecoin/dogecoinparser_test.go +++ b/bchain/coins/dogecoin/dogecoinparser_test.go @@ -9,12 +9,13 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "path/filepath" "reflect" "testing" ) -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -53,14 +54,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -75,24 +76,6 @@ var ( ) func init() { - var ( - addr1, addr2, addr3, addr4 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("DSvXNiqvG42wdteLqh3i6inxgDTs8Y9w2i") - if err == nil { - addr2, err = bchain.NewBaseAddress("DRemF3ZcqJ1PFeM7e7sXzzwQJKR8GNUtwK") - } - if err == nil { - addr3, err = bchain.NewBaseAddress("DJa8bWDrZKu4HgsYRYWuJrvxt6iTYuvXJ6") - } - if err == nil { - addr4, err = bchain.NewBaseAddress("DDTtqnuZ5kfRT5qh2c7sNtqrJmV3iXYdGG") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "01000000016b3c0c53267964120acf7f7e72217e3f463e52ce622f89659f6a6bb8e69a4d91000000006c493046022100a96454237e3a020994534583e28c04757881374bceac89f933ea9ff00b4db259022100fbb757ff7ea4f02c4e42556b2834c61eba1f1af605db089d836a0614d90a3b46012103cebdde6d1046e285df4f48497bc50dc20a4a258ca5b7308cb0a929c9fdadcd9dffffffff0217e823ca7f0200001976a914eef21768a546590993e313c7f3dfadf6a6efa1e888acaddf4cba010000001976a914e0fee2ea29dd9c6c759d8341bd0da4c4f738cced88ac00000000", Blocktime: 1519053456, @@ -111,26 +94,24 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 27478.75452951, - N: 0, + ValueSat: *big.NewInt(2747875452951), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914eef21768a546590993e313c7f3dfadf6a6efa1e888ac", Addresses: []string{ "DSvXNiqvG42wdteLqh3i6inxgDTs8Y9w2i", }, }, - Address: addr1, }, { - Value: 74.20567469, - N: 1, + ValueSat: *big.NewInt(7420567469), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e0fee2ea29dd9c6c759d8341bd0da4c4f738cced88ac", Addresses: []string{ "DRemF3ZcqJ1PFeM7e7sXzzwQJKR8GNUtwK", }, }, - Address: addr2, }, }, } @@ -153,26 +134,24 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 59890867.89818935, - N: 0, + ValueSat: *big.NewInt(5989086789818935), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149355c01ed20057eac9fe0bbf8b07d87e62fe712d88ac", Addresses: []string{ "DJa8bWDrZKu4HgsYRYWuJrvxt6iTYuvXJ6", }, }, - Address: addr3, }, { - Value: 9999998.90000000, - N: 1, + ValueSat: *big.NewInt(999999890000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9145b4f2511c94e4fcaa8f8835b2458f8cb6542ca7688ac", Addresses: []string{ "DDTtqnuZ5kfRT5qh2c7sNtqrJmV3iXYdGG", }, }, - Address: addr4, }, }, } @@ -280,89 +259,111 @@ func Test_UnpackTx(t *testing.T) { } } -var testParseBlockTxs = map[int][]string{ +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ // block without auxpow - 12345: []string{ - "9d1662dcc1443af9999c4fd1d6921b91027b5e2d0d3ebfaa41d84163cb99cad5", - "8284292cedeb0c9c509f9baa235802d52a546e1e9990040d35d018b97ad11cfa", - "3299d93aae5c3d37c795c07150ceaf008aefa5aad3205ea2519f94a35adbbe10", - "3f03016f32b63db48fdc0b17443c2d917ba5e307dcc2fc803feeb21c7219ee1b", - "a889449e9bc618c131c01f564cd309d2217ba1c5731480314795e44f1e02609b", - "29f79d91c10bc311ff5b69fe7ba57101969f68b6391cf0ca67d5f37ca1f0601b", - "b794ebc7c0176c35b125cd8b84a980257cf3dd9cefe2ed47da4ed1d73ee568f3", - "0ec479ba3c954dd422d75c4c5488a6edc3c588deb10ebdbfa8bd8edb7afcfea0", - "f357b6e667dfa456e7988bfa474377df25d0e0bfe07e5f97fc97ea3a0155f031", - "4ff189766f0455721a93d6be27a91eafa750383c800cb053fad2f86c434122d2", - "446d164e2ec4c9f2ac6c499c110735606d949a3625fb849274ac627c033eddbc", - "c489edebd8a2e17fd08f2801f528b95663aaafe15c897d56686423dd430e2d1f", - "3f42a7f1a356897da324d41eed94169c79438212bb9874eea58e9cbaf07481df", - "62c88fdd0fb111676844fcbaebc9e2211a0c990aa7e7529539cb25947a307a1b", - "522c47e315bc1949826339c535d419eb206aec4a332f91dfbd25c206f3c9527b", - "18ea78346e7e34cbdf2d2b6ba1630f8b15f9ef9a940114a3e6ee92d26f96691e", - "43dc0fbd1b9b87bcfc9a51c89457a7b3274855c01d429193aff1181791225f3c", - "d78cdfaadbe5b6b591529cb5c6869866a4cabe46ef82aa835fd2432056b4a383", - "d181759c7a3900ccaf4958f1f25a44949163ceefc306006502efc7a1de6f579e", - "8610b9230188854c7871258163cd1c2db353443d631c5512bff17224a24e95bf", - "e82f40a6bea32122f1d568d427c92708dcb684bdb3035ff3905617230e5ae5b8", - "c50ae6c127f8c346c60e7438fbd10c44c3629f3fe426646db77a2250fb2939f9", - "585202c03894ecaf25188ba4e5447dadd413f2010c2dc2a65c37598dbc6ad907", - "8bd766fde8c65e2f724dad581944dde4e23e4dbb4f7f7faf55bc348923f4d5ee", - "2d2fa25691088181569e508dd8f683b21f2b80ceefb5ccbd6714ebe2a697139f", - "5954622ffc602bec177d61da6c26a68990c42c1886627b218c3ab0e9e3491f4a", - "01b634bc53334df1cd9f04522729a34d811c418c2535144c3ed156cbc319e43e", - "c429a6c8265482b2d824af03afe1c090b233a856f243791485cb4269f2729649", - "dbe79231b916b6fb47a91ef874f35150270eb571af60c2d640ded92b41749940", - "1c396493a8dfd59557052b6e8643123405894b64f48b2eb6eb7a003159034077", - "2e2816ffb7bf1378f11acf5ba30d498efc8fd219d4b67a725e8254ce61b1b7ee", + 12345: testBlock{ + size: 8582, + time: 1387104223, + txs: []string{ + "9d1662dcc1443af9999c4fd1d6921b91027b5e2d0d3ebfaa41d84163cb99cad5", + "8284292cedeb0c9c509f9baa235802d52a546e1e9990040d35d018b97ad11cfa", + "3299d93aae5c3d37c795c07150ceaf008aefa5aad3205ea2519f94a35adbbe10", + "3f03016f32b63db48fdc0b17443c2d917ba5e307dcc2fc803feeb21c7219ee1b", + "a889449e9bc618c131c01f564cd309d2217ba1c5731480314795e44f1e02609b", + "29f79d91c10bc311ff5b69fe7ba57101969f68b6391cf0ca67d5f37ca1f0601b", + "b794ebc7c0176c35b125cd8b84a980257cf3dd9cefe2ed47da4ed1d73ee568f3", + "0ec479ba3c954dd422d75c4c5488a6edc3c588deb10ebdbfa8bd8edb7afcfea0", + "f357b6e667dfa456e7988bfa474377df25d0e0bfe07e5f97fc97ea3a0155f031", + "4ff189766f0455721a93d6be27a91eafa750383c800cb053fad2f86c434122d2", + "446d164e2ec4c9f2ac6c499c110735606d949a3625fb849274ac627c033eddbc", + "c489edebd8a2e17fd08f2801f528b95663aaafe15c897d56686423dd430e2d1f", + "3f42a7f1a356897da324d41eed94169c79438212bb9874eea58e9cbaf07481df", + "62c88fdd0fb111676844fcbaebc9e2211a0c990aa7e7529539cb25947a307a1b", + "522c47e315bc1949826339c535d419eb206aec4a332f91dfbd25c206f3c9527b", + "18ea78346e7e34cbdf2d2b6ba1630f8b15f9ef9a940114a3e6ee92d26f96691e", + "43dc0fbd1b9b87bcfc9a51c89457a7b3274855c01d429193aff1181791225f3c", + "d78cdfaadbe5b6b591529cb5c6869866a4cabe46ef82aa835fd2432056b4a383", + "d181759c7a3900ccaf4958f1f25a44949163ceefc306006502efc7a1de6f579e", + "8610b9230188854c7871258163cd1c2db353443d631c5512bff17224a24e95bf", + "e82f40a6bea32122f1d568d427c92708dcb684bdb3035ff3905617230e5ae5b8", + "c50ae6c127f8c346c60e7438fbd10c44c3629f3fe426646db77a2250fb2939f9", + "585202c03894ecaf25188ba4e5447dadd413f2010c2dc2a65c37598dbc6ad907", + "8bd766fde8c65e2f724dad581944dde4e23e4dbb4f7f7faf55bc348923f4d5ee", + "2d2fa25691088181569e508dd8f683b21f2b80ceefb5ccbd6714ebe2a697139f", + "5954622ffc602bec177d61da6c26a68990c42c1886627b218c3ab0e9e3491f4a", + "01b634bc53334df1cd9f04522729a34d811c418c2535144c3ed156cbc319e43e", + "c429a6c8265482b2d824af03afe1c090b233a856f243791485cb4269f2729649", + "dbe79231b916b6fb47a91ef874f35150270eb571af60c2d640ded92b41749940", + "1c396493a8dfd59557052b6e8643123405894b64f48b2eb6eb7a003159034077", + "2e2816ffb7bf1378f11acf5ba30d498efc8fd219d4b67a725e8254ce61b1b7ee", + }, }, // 1st block with auxpow - 371337: []string{ - "4547b14bc16db4184fa9f141d645627430dd3dfa662d0e6f418fba497091da75", - "a965dba2ed06827ed9a24f0568ec05b73c431bc7f0fb6913b144e62db7faa519", - "5e3ab18cb7ba3abc44e62fb3a43d4c8168d00cf0a2e0f8dbeb2636bb9a212d12", - "f022935ac7c4c734bd2c9c6a780f8e7280352de8bd358d760d0645b7fe734a93", - "ec063cc8025f9f30a6ed40fc8b1fe63b0cbd2ea2c62664eb26b365e6243828ca", - "02c16e3389320da3e77686d39773dda65a1ecdf98a2ef9cfb938c9f4b58f7a40", + 371337: testBlock{ + size: 1704, + time: 1410464577, + txs: []string{ + "4547b14bc16db4184fa9f141d645627430dd3dfa662d0e6f418fba497091da75", + "a965dba2ed06827ed9a24f0568ec05b73c431bc7f0fb6913b144e62db7faa519", + "5e3ab18cb7ba3abc44e62fb3a43d4c8168d00cf0a2e0f8dbeb2636bb9a212d12", + "f022935ac7c4c734bd2c9c6a780f8e7280352de8bd358d760d0645b7fe734a93", + "ec063cc8025f9f30a6ed40fc8b1fe63b0cbd2ea2c62664eb26b365e6243828ca", + "02c16e3389320da3e77686d39773dda65a1ecdf98a2ef9cfb938c9f4b58f7a40", + }, }, // block with auxpow - 567890: []string{ - "db20feea53be1f60848a66604d5bca63df62de4f6c66220f9c84436d788625a8", - "cf7e9e27c0f56f0b100eaf5c776ce106025e3412bd5927c6e1ce575500e24eaa", - "af84e010c1cf0bd927740d08e5e8163db45397b70f00df07aea5339c14d5f3aa", - "7362e25e8131255d101e5d874e6b6bb2faa7a821356cb041f1843d0901dffdbd", - "3b875344302e8893f6d5c9e7269d806ed27217ec67944940ae9048fc619bdae9", - "e3b95e269b7c251d87e8e241ea2a08a66ec14d12a1012762be368b3db55471e3", - "6ba3f95a37bcab5d0cb5b8bd2fe48040db0a6ae390f320d6dcc8162cc096ff8f", - "3211ccc66d05b10959fa6e56d1955c12368ea52b40303558b254d7dc22570382", - "54c1b279e78b924dfa15857c80131c3ddf835ab02f513dc03aa514f87b680493", + 567890: testBlock{ + size: 3833, + time: 1422855443, + txs: []string{ + "db20feea53be1f60848a66604d5bca63df62de4f6c66220f9c84436d788625a8", + "cf7e9e27c0f56f0b100eaf5c776ce106025e3412bd5927c6e1ce575500e24eaa", + "af84e010c1cf0bd927740d08e5e8163db45397b70f00df07aea5339c14d5f3aa", + "7362e25e8131255d101e5d874e6b6bb2faa7a821356cb041f1843d0901dffdbd", + "3b875344302e8893f6d5c9e7269d806ed27217ec67944940ae9048fc619bdae9", + "e3b95e269b7c251d87e8e241ea2a08a66ec14d12a1012762be368b3db55471e3", + "6ba3f95a37bcab5d0cb5b8bd2fe48040db0a6ae390f320d6dcc8162cc096ff8f", + "3211ccc66d05b10959fa6e56d1955c12368ea52b40303558b254d7dc22570382", + "54c1b279e78b924dfa15857c80131c3ddf835ab02f513dc03aa514f87b680493", + }, }, // recent block - 2264125: []string{ - "76f0126562c99e020b5fba41b68dd8141a4f21eef62012b76a1e0635092045e9", - "7bb6688bec16de94014574e3e1d3f6f5fb956530d6b179b28db367f1fd8ae099", - "d7e2ee30c3d179ac896651fc09c1396333f41d952d008af8d5d6665cbea377bf", - "8e4783878df782003c43d014fcbb9c57d2034dfd1d9fcd7319bb1a9f501dbbb7", - "8d2a4ae226b6f23eea545957be5d71c68cd08674d96a3502d4ca21ffadacb5a9", - "a0da2b49de881133655c54b1b5c23af443a71c2b937e2d9bbdf3f498247e6b7b", - "c780a19b9cf46ed70b53c5d5722e8d33951211a4051cb165b25fb0c22a4ae1ff", - "ce29c2644d642bb4fedd09d0840ed98c9945bf292967fede8fcc6b26054b4058", - "a360b0566f68c329e2757918f67ee6421d3d76f70f1b452cdd32266805986119", - "17e85bd33cc5fb5035e489c5188979f45e75e92d14221eca937e14f5f7d7b074", - "3973eb930fd2d0726abbd81912eae645384268cd3500b9ec84d806fdd65a426a", - "b91cc1c98e5c77e80eec9bf93e86af27f810b00dfbce3ee2646758797a28d5f2", - "1a8c7bd3389dcbbc1133ee600898ed9e082f7a9c75f9eb52f33940ed7c2247ef", - "9b1782449bbd3fc3014c363167777f7bdf41f5ef6db192fbda784b29603911b0", - "afab4bcdc1a32891d638579c3029ae49ee72be3303425c6d62e1f8eaebe0ce18", - "5f839f9cd5293c02ff4f7cf5589c53dec52adb42a077599dc7a2c5842a156ca9", - "756d2dfd1d2872ba2531fae3b8984008506871bec41d19cb299f5e0f216cfb9b", - "6aa82514ab7a9cc624fabf3d06ccbd46ecb4009b3c784768e6243d7840d4bf93", - "d1430b3f7ecf147534796c39ba631ea22ac03530e25b9428367c0dc381b10863", - "2aeb69b1eb9eef8039da6b97d7851e46f57325851e6998ef5a84fc9a826c2c74", - "fc61d13eef806af8da693cfa621fe92110694f1514567b186a35c54e7ef4a188", - "a02dd44e60ba62fa00c83a67116f8079bf71062939b207bee0808cb98b30cf22", - "279f97cfc606fe62777b44614ff28675ce661687904e068e3ec79f619c4fdae7", - "d515d271849717b091a9c46bf11c47efb9d975e72b668c137786a208cf0a9739", - "a800da44e6eed944043561fe22ee0a6e11341e6bc1a8ec2789b83930cc9b170e", + 2264125: testBlock{ + size: 8531, + time: 1529099968, + txs: []string{ + "76f0126562c99e020b5fba41b68dd8141a4f21eef62012b76a1e0635092045e9", + "7bb6688bec16de94014574e3e1d3f6f5fb956530d6b179b28db367f1fd8ae099", + "d7e2ee30c3d179ac896651fc09c1396333f41d952d008af8d5d6665cbea377bf", + "8e4783878df782003c43d014fcbb9c57d2034dfd1d9fcd7319bb1a9f501dbbb7", + "8d2a4ae226b6f23eea545957be5d71c68cd08674d96a3502d4ca21ffadacb5a9", + "a0da2b49de881133655c54b1b5c23af443a71c2b937e2d9bbdf3f498247e6b7b", + "c780a19b9cf46ed70b53c5d5722e8d33951211a4051cb165b25fb0c22a4ae1ff", + "ce29c2644d642bb4fedd09d0840ed98c9945bf292967fede8fcc6b26054b4058", + "a360b0566f68c329e2757918f67ee6421d3d76f70f1b452cdd32266805986119", + "17e85bd33cc5fb5035e489c5188979f45e75e92d14221eca937e14f5f7d7b074", + "3973eb930fd2d0726abbd81912eae645384268cd3500b9ec84d806fdd65a426a", + "b91cc1c98e5c77e80eec9bf93e86af27f810b00dfbce3ee2646758797a28d5f2", + "1a8c7bd3389dcbbc1133ee600898ed9e082f7a9c75f9eb52f33940ed7c2247ef", + "9b1782449bbd3fc3014c363167777f7bdf41f5ef6db192fbda784b29603911b0", + "afab4bcdc1a32891d638579c3029ae49ee72be3303425c6d62e1f8eaebe0ce18", + "5f839f9cd5293c02ff4f7cf5589c53dec52adb42a077599dc7a2c5842a156ca9", + "756d2dfd1d2872ba2531fae3b8984008506871bec41d19cb299f5e0f216cfb9b", + "6aa82514ab7a9cc624fabf3d06ccbd46ecb4009b3c784768e6243d7840d4bf93", + "d1430b3f7ecf147534796c39ba631ea22ac03530e25b9428367c0dc381b10863", + "2aeb69b1eb9eef8039da6b97d7851e46f57325851e6998ef5a84fc9a826c2c74", + "fc61d13eef806af8da693cfa621fe92110694f1514567b186a35c54e7ef4a188", + "a02dd44e60ba62fa00c83a67116f8079bf71062939b207bee0808cb98b30cf22", + "279f97cfc606fe62777b44614ff28675ce661687904e068e3ec79f619c4fdae7", + "d515d271849717b091a9c46bf11c47efb9d975e72b668c137786a208cf0a9739", + "a800da44e6eed944043561fe22ee0a6e11341e6bc1a8ec2789b83930cc9b170e", + }, }, } @@ -389,7 +390,7 @@ func helperLoadBlock(t *testing.T, height int) []byte { func TestParseBlock(t *testing.T) { p := NewDogecoinParser(GetChainParams("main"), &btc.Configuration{}) - for height, txs := range testParseBlockTxs { + for height, tb := range testParseBlockTxs { b := helperLoadBlock(t, height) blk, err := p.ParseBlock(b) @@ -397,11 +398,19 @@ func TestParseBlock(t *testing.T) { t.Fatal(err) } - if len(blk.Txs) != len(txs) { - t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(txs)) + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) } - for ti, tx := range txs { + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { if blk.Txs[ti].Txid != tx { t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) } diff --git a/bchain/coins/dogecoin/dogecoinrpc.go b/bchain/coins/dogecoin/dogecoinrpc.go index d0ebb31c..1fc21618 100644 --- a/bchain/coins/dogecoin/dogecoinrpc.go +++ b/bchain/coins/dogecoin/dogecoinrpc.go @@ -24,6 +24,7 @@ func NewDogecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *DogecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *DogecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index fe361258..d76aa6bf 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -22,7 +22,10 @@ type EthereumParser struct { // NewEthereumParser returns new EthereumParser instance func NewEthereumParser() *EthereumParser { - return &EthereumParser{&bchain.BaseParser{AddressFactory: bchain.NewBaseAddress}} + return &EthereumParser{&bchain.BaseParser{ + BlockAddressesToKeep: 0, + AmountDecimalPoint: 18, + }} } type rpcTransaction struct { @@ -64,7 +67,6 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma txid := ethHashToHash(tx.Hash) var ( fa, ta []string - addr bchain.Address err error ) if len(tx.From) > 2 { @@ -72,10 +74,6 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } if len(tx.To) > 2 { ta = []string{tx.To} - addr, err = p.AddressFactory(tx.To) - if err != nil { - return nil, err - } } // temporarily, the complete rpcTransaction without BlockHash is marshalled and hex encoded to bchain.Tx.Hex bh := tx.BlockHash @@ -86,6 +84,10 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma } tx.BlockHash = bh h := hex.EncodeToString(b) + vs, err := hexutil.DecodeBig(tx.Value) + if err != nil { + return nil, err + } return &bchain.Tx{ Blocktime: blocktime, Confirmations: confirmations, @@ -105,32 +107,31 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma }, Vout: []bchain.Vout{ { - N: 0, // there is always up to one To address - // Value - cannot be set, it does not fit precisely to float64 + N: 0, // there is always up to one To address + ValueSat: *vs, ScriptPubKey: bchain.ScriptPubKey{ // Hex Addresses: ta, }, - Address: addr, }, }, }, nil } -// GetAddrIDFromVout returns internal address representation of given transaction output -func (p *EthereumParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { +// GetAddrDescFromVout returns internal address representation of given transaction output +func (p *EthereumParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { if len(output.ScriptPubKey.Addresses) != 1 { return nil, bchain.ErrAddressMissing } - return p.GetAddrIDFromAddress(output.ScriptPubKey.Addresses[0]) + return p.GetAddrDescFromAddress(output.ScriptPubKey.Addresses[0]) } func has0xPrefix(s string) bool { return len(s) >= 2 && s[0] == '0' && (s[1]|32) == 'x' } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *EthereumParser) GetAddrIDFromAddress(address string) ([]byte, error) { +// GetAddrDescFromAddress returns internal address representation of given address +func (p *EthereumParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { // github.com/ethereum/go-ethereum/common.HexToAddress does not handle address errors, using own decoding if has0xPrefix(address) { address = address[2:] @@ -144,6 +145,16 @@ func (p *EthereumParser) GetAddrIDFromAddress(address string) ([]byte, error) { return hex.DecodeString(address) } +// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable +func (p *EthereumParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { + return []string{hexutil.Encode(addrDesc)}, true, nil +} + +// GetScriptFromAddrDesc returns output script for given address descriptor +func (p *EthereumParser) GetScriptFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]byte, error) { + return addrDesc, nil +} + func hexDecode(s string) ([]byte, error) { b, err := hexutil.Decode(s) if err != nil && err != hexutil.ErrEmptyString { @@ -286,9 +297,3 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) { func (p *EthereumParser) IsUTXOChain() bool { return false } - -// KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column -// do not use the blockaddresses for eth -func (p *EthereumParser) KeepBlockAddresses() int { - return 0 -} diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index c1f4620f..2a7ee8bf 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -5,11 +5,12 @@ package eth import ( "blockbook/bchain" "encoding/hex" + "math/big" "reflect" "testing" ) -func TestEthParser_GetAddrIDFromAddress(t *testing.T) { +func TestEthParser_GetAddrDescFromAddress(t *testing.T) { type args struct { address string } @@ -50,19 +51,68 @@ func TestEthParser_GetAddrIDFromAddress(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { p := NewEthereumParser() - got, err := p.GetAddrIDFromAddress(tt.args.address) + got, err := p.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("EthParser.GetAddrIDFromAddress() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("EthParser.GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("EthParser.GetAddrIDFromAddress() = %v, want %v", h, tt.want) + t.Errorf("EthParser.GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } +var ( + testTx1, testTx2 bchain.Tx + testTxPacked1 = "08aebf0a1205012a05f20018a0f73622081234567890abcdef2a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f" + testTxPacked2 = "08ece40212050430e234001888a4012201213220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915" +) + +func init() { + + testTx1 = bchain.Tx{ + Blocktime: 1521515026, + Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307831323334353637383930616263646566222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", + Time: 1521515026, + Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + Vin: []bchain.Vin{ + { + Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(1311768467294899695), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, + }, + }, + }, + } + + testTx2 = bchain.Tx{ + Blocktime: 1521533434, + Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a2230783231222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", + Time: 1521533434, + Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", + Vin: []bchain.Vin{ + { + Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(33), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, + }, + }, + }, + } +} + func TestEthereumParser_PackTx(t *testing.T) { type args struct { tx *bchain.Tx @@ -77,61 +127,27 @@ func TestEthereumParser_PackTx(t *testing.T) { wantErr bool }{ { - name: "with 0x prefix", + name: "1", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - }, - }, - }, + tx: &testTx1, height: 2870000, blockTime: 1521515026, }, - want: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f", + want: testTxPacked1, }, { - name: "without 0x prefix", + name: "2", args: args{ - tx: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - }, - }, - }, + tx: &testTx2, height: 2871048, blockTime: 1521533434, }, - want: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915", + want: testTxPacked2, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime) if (err != nil) != tt.wantErr { t.Errorf("EthereumParser.PackTx() error = %v, wantErr %v", err, tt.wantErr) @@ -146,18 +162,6 @@ func TestEthereumParser_PackTx(t *testing.T) { } func TestEthereumParser_UnpackTx(t *testing.T) { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("0x682b7903a11098cf770c7aef4aa02a85b3f3601a") - if err == nil { - addr2, err = bchain.NewBaseAddress("0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f") - } - if err != nil { - panic(err) - } - type args struct { hex string } @@ -170,57 +174,21 @@ func TestEthereumParser_UnpackTx(t *testing.T) { wantErr bool }{ { - name: "1", - args: args{hex: "08aebf0a1205012a05f20018a0f7362a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f"}, - want: &bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - Address: addr1, - }, - }, - }, + name: "1", + args: args{hex: testTxPacked1}, + want: &testTx1, want1: 2870000, }, { - name: "1", - args: args{hex: "08ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915"}, - want: &bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a22307831626330313539643533306536303030222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, - Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", - Vin: []bchain.Vin{ - { - Addresses: []string{"0x3e3a3d69dc66ba10737f531ed088954a9ec89d97"}, - }, - }, - Vout: []bchain.Vout{ - { - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, - }, - Address: addr2, - }, - }, - }, + name: "2", + args: args{hex: testTxPacked2}, + want: &testTx2, want1: 2871048, }, } + p := NewEthereumParser() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() b, err := hex.DecodeString(tt.args.hex) if err != nil { panic(err) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index a182a6fd..6bdda516 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "math/big" + "strconv" "sync" "time" @@ -260,16 +261,23 @@ func (b *EthereumRPC) GetSubversion() string { return "" } -// GetBlockChainInfo returns the NetworkID of the ethereum network -func (b *EthereumRPC) GetBlockChainInfo() (string, error) { +// GetChainInfo returns information about the connected backend +func (b *EthereumRPC) GetChainInfo() (*bchain.ChainInfo, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() - id, err := b.client.NetworkID(ctx) if err != nil { - return "", err + return nil, err } - return id.String(), nil + rv := &bchain.ChainInfo{} + idi := int(id.Uint64()) + if idi == 1 { + rv.Chain = "mainnet" + } else { + rv.Chain = "testnet " + strconv.Itoa(idi) + } + // TODO - return more information about the chain + return rv, nil } func (b *EthereumRPC) getBestHeader() (*ethtypes.Header, error) { @@ -336,9 +344,9 @@ func (b *EthereumRPC) ethHeaderToBlockHeader(h *ethtypes.Header) (*bchain.BlockH Hash: ethHashToHash(h.Hash()), Height: uint32(hn), Confirmations: int(c), + Time: int64(h.Time.Uint64()), // Next // Prev - }, nil } @@ -408,6 +416,8 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error return nil, errors.Annotatef(fmt.Errorf("server returned empty transaction list but block header indicates transactions"), "hash %v, height %v", hash, height) } bbh, err := b.ethHeaderToBlockHeader(head) + // TODO - this is probably not the correct size + bbh.Size = len(raw) btxs := make([]bchain.Tx, len(body.Transactions)) for i, tx := range body.Transactions { btx, err := b.Parser.ethTxToTx(&tx, int64(head.Time.Uint64()), uint32(bbh.Confirmations)) @@ -423,6 +433,12 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error return &bbk, nil } +// GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids +func (b *EthereumRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + // TODO - implement + return nil, errors.New("Not implemented yet") +} + // GetTransactionForMempool returns a transaction by the transaction ID. // It could be optimized for mempool, i.e. without block time and confirmations func (b *EthereumRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { @@ -497,13 +513,13 @@ func (b *EthereumRPC) GetMempool() ([]string, error) { return body.Transactions, nil } -// EstimateFee returns fee estimation. -func (b *EthereumRPC) EstimateFee(blocks int) (float64, error) { +// EstimateFee returns fee estimation +func (b *EthereumRPC) EstimateFee(blocks int) (big.Int, error) { return b.EstimateSmartFee(blocks, true) } -// EstimateSmartFee returns fee estimation. -func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { +// EstimateSmartFee returns fee estimation +func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() // TODO - what parameters of msg to use to get better estimate, maybe more data from the wallet are needed @@ -512,10 +528,12 @@ func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (float64, To: &a, } g, err := b.client.EstimateGas(ctx, msg) + var r big.Int if err != nil { - return 0, err + return r, err } - return float64(g), nil + r.SetUint64(g) + return r, nil } // SendRawTransaction sends raw transaction. @@ -543,10 +561,16 @@ func (b *EthereumRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, er return b.Mempool.Resync(onNewTxAddr) } +// GetMempoolTransactions returns slice of mempool transactions for given address func (b *EthereumRPC) GetMempoolTransactions(address string) ([]string, error) { return b.Mempool.GetTransactions(address) } +// GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor +func (b *EthereumRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, error) { + return b.Mempool.GetAddrDescTransactions(addrDesc) +} + func (b *EthereumRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { return nil, errors.New("GetMempoolEntry: not implemented") } diff --git a/bchain/coins/eth/ethrpc_test.go b/bchain/coins/eth/ethrpc_test.go deleted file mode 100644 index fab3349f..00000000 --- a/bchain/coins/eth/ethrpc_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// +build integration - -package eth - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewEthereumRPC(cfg, nil) - if err != nil { - return nil, err - } - return c, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Ethereum Testnet", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestEthRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestEthRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestEthRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestEthRPC_GetBestBlockHash(t *testing.T) { - rpcTest.TestGetBestBlockHash(t) -} - -func TestEthRPC_GetBestBlockHeight(t *testing.T) { - rpcTest.TestGetBestBlockHeight(t) -} - -func TestEthRPC_GetBlockHeader(t *testing.T) { - rpcTest.TestGetBlockHeader(t) -} - -func TestEthRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} diff --git a/bchain/coins/litecoin/litecoinparser.go b/bchain/coins/litecoin/litecoinparser.go index 3cb06f34..7a60d2ac 100644 --- a/bchain/coins/litecoin/litecoinparser.go +++ b/bchain/coins/litecoin/litecoinparser.go @@ -3,8 +3,8 @@ package litecoin import ( "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -18,17 +18,17 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 48 - MainNetParams.ScriptHashAddrID = 50 + MainNetParams.PubKeyHashAddrID = []byte{48} + MainNetParams.ScriptHashAddrID = []byte{50} MainNetParams.Bech32HRPSegwit = "ltc" TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic - TestNetParams.PubKeyHashAddrID = 111 - TestNetParams.ScriptHashAddrID = 58 + TestNetParams.PubKeyHashAddrID = []byte{111} + TestNetParams.ScriptHashAddrID = []byte{58} TestNetParams.Bech32HRPSegwit = "tltc" err := chaincfg.Register(&MainNetParams) @@ -53,6 +53,9 @@ func NewLitecoinParser(params *chaincfg.Params, c *btc.Configuration) *LitecoinP // GetChainParams contains network parameters for the main Litecoin network, // and the test Litecoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/litecoin/litecoinparser_test.go b/bchain/coins/litecoin/litecoinparser_test.go index 11c55f1e..1e046bcd 100644 --- a/bchain/coins/litecoin/litecoinparser_test.go +++ b/bchain/coins/litecoin/litecoinparser_test.go @@ -6,11 +6,12 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) -func TestAddressToOutputScript_Testnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Testnet(t *testing.T) { type args struct { address string } @@ -43,20 +44,20 @@ func TestAddressToOutputScript_Testnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -107,14 +108,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -127,18 +128,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("LMgENNXzzuPxp7vfMjDrCU44bsmrEMgqvc") - if err == nil { - addr2, err = bchain.NewBaseAddress("LV1ByjbJNFTHyFQqwqwdJXKJznYDzXzg4B") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "02000000031e1977dc524bec5929e95d8d0946812944b7b5bda12f5b99fdf557773f2ee65e0100000000ffffffff8a398e44546dce0245452b90130e86832b21fd68f26662bc33aeb7c6c115d23c1900000000ffffffffb807ab93a7fcdff7af6d24581a4a18aa7c1db1ebecba2617a6805b009513940f0c00000000ffffffff020001a04a000000001976a9141ae882e788091732da6910595314447c9e38bd8d88ac27440f00000000001976a9146b474cbf0f6004329b630bdd4798f2c23d1751b688ac00000000", Blocktime: 1519053456, @@ -173,26 +162,24 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 12.52000000, - N: 0, + ValueSat: *big.NewInt(1252000000), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9141ae882e788091732da6910595314447c9e38bd8d88ac", Addresses: []string{ "LMgENNXzzuPxp7vfMjDrCU44bsmrEMgqvc", }, }, - Address: addr1, }, { - Value: 0.01000487, - N: 1, + ValueSat: *big.NewInt(1000487), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9146b474cbf0f6004329b630bdd4798f2c23d1751b688ac", Addresses: []string{ "LV1ByjbJNFTHyFQqwqwdJXKJznYDzXzg4B", }, }, - Address: addr2, }, }, } diff --git a/bchain/coins/litecoin/litecoinrpc.go b/bchain/coins/litecoin/litecoinrpc.go index 0d2ad15f..e7345251 100644 --- a/bchain/coins/litecoin/litecoinrpc.go +++ b/bchain/coins/litecoin/litecoinrpc.go @@ -24,6 +24,7 @@ func NewLitecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *LitecoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *LitecoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/monacoin/monacoinparser.go b/bchain/coins/monacoin/monacoinparser.go index 1268e843..d5a9de37 100644 --- a/bchain/coins/monacoin/monacoinparser.go +++ b/bchain/coins/monacoin/monacoinparser.go @@ -1,10 +1,11 @@ package monacoin import ( + "blockbook/bchain" "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" monacoinCfg "github.com/wakiyamap/monad/chaincfg" "github.com/wakiyamap/monad/txscript" monacoinWire "github.com/wakiyamap/monad/wire" @@ -25,11 +26,11 @@ var ( MonaTestParams monacoinCfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 50 - MainNetParams.ScriptHashAddrID = 55 + MainNetParams.PubKeyHashAddrID = []byte{50} + MainNetParams.ScriptHashAddrID = []byte{55} MainNetParams.Bech32HRPSegwit = "mona" MonaMainParams = monacoinCfg.MainNetParams MonaMainParams.Net = MonaMainMagic @@ -39,8 +40,8 @@ func init() { TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic - TestNetParams.PubKeyHashAddrID = 111 - TestNetParams.ScriptHashAddrID = 117 + TestNetParams.PubKeyHashAddrID = []byte{111} + TestNetParams.ScriptHashAddrID = []byte{117} TestNetParams.Bech32HRPSegwit = "tmona" MonaTestParams = monacoinCfg.TestNet4Params MonaTestParams.Net = MonaTestMagic @@ -70,6 +71,9 @@ func NewMonacoinParser(params *chaincfg.Params, c *btc.Configuration) *MonacoinP // GetChainParams contains network parameters for the main Monacoin network, // and the test Monacoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams @@ -89,13 +93,13 @@ func GetMonaChainParams(chain string) *monacoinCfg.Params { } } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *MonacoinParser) GetAddrIDFromAddress(address string) ([]byte, error) { - return p.AddressToOutputScript(address) +// GetAddrDescFromAddress returns internal address representation (descriptor) of given address +func (p *MonacoinParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { + return p.addressToOutputScript(address) } -// AddressToOutputScript converts monacoin address to ScriptPubKey -func (p *MonacoinParser) AddressToOutputScript(address string) ([]byte, error) { +// addressToOutputScript converts monacoin address to ScriptPubKey +func (p *MonacoinParser) addressToOutputScript(address string) ([]byte, error) { switch p.Params.Net { case MainnetMagic: da, err := monautil.DecodeAddress(address, &MonaMainParams) diff --git a/bchain/coins/monacoin/monacoin_test.go b/bchain/coins/monacoin/monacoinparser_test.go similarity index 75% rename from bchain/coins/monacoin/monacoin_test.go rename to bchain/coins/monacoin/monacoinparser_test.go index 207f3378..7c95d9a2 100644 --- a/bchain/coins/monacoin/monacoin_test.go +++ b/bchain/coins/monacoin/monacoinparser_test.go @@ -1,4 +1,4 @@ -// build unittest +// +build unittest package monacoin @@ -6,11 +6,12 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) -func TestAddressToOutputScript_Testnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Testnet(t *testing.T) { type args struct { address string } @@ -43,20 +44,20 @@ func TestAddressToOutputScript_Testnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -107,14 +108,89 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) + } + }) + } +} + +func Test_GetAddressesFromAddrDesc(t *testing.T) { + type args struct { + script string + } + tests := []struct { + name string + args args + want []string + want2 bool + wantErr bool + }{ + { + name: "P2PKH", + args: args{script: "76a91451dadacc7021440cbe4ca148a5db563b329b4c0388ac"}, + want: []string{"MFMy9FwJsV6HiN5eZDqDETw4pw52q3UGrb"}, + want2: true, + wantErr: false, + }, + { + name: "P2SH", + args: args{script: "a9146449f568c9cd2378138f2636e1567112a184a9e887"}, + want: []string{"PHjTKtgYLTJ9D2Bzw2f6xBB41KBm2HeGfg"}, + want2: true, + wantErr: false, + }, + { + name: "P2WPKH", + args: args{script: "0014a96d3cef194f469b33801f868ec9bc89a8831c22"}, + want: []string{"mona1q49knemcefarfkvuqr7rgajdu3x5gx8pzdnurgq"}, + want2: true, + wantErr: false, + }, + { + name: "P2WSH", + args: args{script: "002009d27aa88e70cb7a0da620908c9bc08ac6c633bd1a61036312e514396aeb4893"}, + want: []string{"mona1qp8f842ywwr9h5rdxyzggex7q3trvvvaarfssxccju52rj6htfzfsqr79j2"}, + want2: true, + wantErr: false, + }, + { + name: "OP_RETURN ascii", + args: args{script: "6a0461686f6a"}, + want: []string{"OP_RETURN (ahoj)"}, + want2: false, + wantErr: false, + }, + { + name: "OP_RETURN hex", + args: args{script: "6a072020f1686f6a20"}, + want: []string{"OP_RETURN 2020f1686f6a20"}, + want2: false, + wantErr: false, + }, + } + + parser := NewMonacoinParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.script) + got, got2, err := parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("outputScriptToAddresses() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) } }) } @@ -127,18 +203,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("MWpWpANNQRskQHcuY5ZQpN4BVynQxmSxRb") - if err == nil { - addr2, err = bchain.NewBaseAddress("MGtFpCVyKEHNtpVNesxPMxYuQayoEBX5yZ") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "0200000003e44ef4e5fe2e4345f1e1340afe396c780773e3834a5bffb153a2faf510e2845e000000006a47304402205ebd735621eaaf512441998727a37e99be94e5ecded54601ea3eebac9282bc2502207d48da44e1c883579c6cd8c2b8ccfb5380e5ac71affe70b475d2b558e0f7bd4b01210391f72b34c04855ce16b97dd79b0ba78fc4b26f40abce853c33788e348cb79c3bfeffffff0ad690a74c43c0df9527c516d26e31fa47e15471a2ead65757b672522888e920010000006b48304502210091a473124bf506edbb095951aa1a32c76bea7eba4020ae2858314961b1a83de602205c3818e517cf830a95a1208fc84aa343faaeeaaa96eab76238379769598ab2d40121038c217e5de8e375ed6cf648e96ec6bfb9e0fbcf5ae3945a5ea60d16919d9c8b68feffffffb9aa4aed4ad4c4b95419e132a43db34aa03a7ec35ef0beecdd627f9ca07bda03010000006a47304402204906d973ac9b4786403f8f8fc2b2ad2e6745ea01a93336b4b67af1d7d1b625cc022016820be905ffd6e11949da79e7a1c7eb97939421a04e0645c8caef8fc585f7ca012102b5f647c4eb677e952913c0b6934c12b29dc50afba8b558b1677ffd2d78c84a88feffffff02f6da4601000000001976a914fb69fe6dcfe88557dc0ce0ea65bd7cf02f5e4f5b88ac8bfd8c57000000001976a914628d603ac50d656e3311ff0cd5490b4c5cdd92ea88ac25fd1400", Blocktime: 1530902705, @@ -173,26 +237,24 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.21420790, - N: 0, + ValueSat: *big.NewInt(21420790), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914fb69fe6dcfe88557dc0ce0ea65bd7cf02f5e4f5b88ac", Addresses: []string{ "MWpWpANNQRskQHcuY5ZQpN4BVynQxmSxRb", }, }, - Address: addr1, }, { - Value: 14.68857739, - N: 1, + ValueSat: *big.NewInt(1468857739), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914628d603ac50d656e3311ff0cd5490b4c5cdd92ea88ac", Addresses: []string{ "MGtFpCVyKEHNtpVNesxPMxYuQayoEBX5yZ", }, }, - Address: addr2, }, }, } diff --git a/bchain/coins/monacoin/monacoinrpc.go b/bchain/coins/monacoin/monacoinrpc.go index 7a522a34..dacddb4c 100644 --- a/bchain/coins/monacoin/monacoinrpc.go +++ b/bchain/coins/monacoin/monacoinrpc.go @@ -24,6 +24,7 @@ func NewMonacoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *MonacoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *MonacoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/monacoin/monacoinrpc_test.go b/bchain/coins/monacoin/monacoinrpc_test.go deleted file mode 100644 index 4a4a7f5c..00000000 --- a/bchain/coins/monacoin/monacoinrpc_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// +build integration - -package monacoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewMonacoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*MonacoinRPC) - cli.Parser = NewMonacoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Monacoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestMonacoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestMonacoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestMonacoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestMonacoinRPC_GetTransactionForMempool(t *testing.T) { - rpcTest.TestGetTransactionForMempool(t) -} - -func TestMonacoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestMonacoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestMonacoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestMonacoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} diff --git a/bchain/coins/namecoin/namecoinparser.go b/bchain/coins/namecoin/namecoinparser.go index 344dbeb4..bca57e55 100644 --- a/bchain/coins/namecoin/namecoinparser.go +++ b/bchain/coins/namecoin/namecoinparser.go @@ -6,8 +6,8 @@ import ( "blockbook/bchain/coins/utils" "bytes" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -18,11 +18,11 @@ var ( MainNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 52 - MainNetParams.ScriptHashAddrID = 13 + MainNetParams.PubKeyHashAddrID = []byte{52} + MainNetParams.ScriptHashAddrID = []byte{13} err := chaincfg.Register(&MainNetParams) if err != nil { @@ -43,6 +43,9 @@ func NewNamecoinParser(params *chaincfg.Params, c *btc.Configuration) *NamecoinP // GetChainParams contains network parameters for the main Namecoin network, // and the test Namecoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { default: return &MainNetParams @@ -75,5 +78,11 @@ func (p *NamecoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: h.Timestamp.Unix(), + }, + Txs: txs, + }, nil } diff --git a/bchain/coins/namecoin/namecoinparser_test.go b/bchain/coins/namecoin/namecoinparser_test.go index 0f22015d..8eff0e00 100644 --- a/bchain/coins/namecoin/namecoinparser_test.go +++ b/bchain/coins/namecoin/namecoinparser_test.go @@ -13,7 +13,7 @@ import ( "testing" ) -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -40,24 +40,34 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } } -var testParseBlockTxs = map[int][]string{ - 40000: []string{ - "e193a821393b20b99f4a4e05a481368ef8a8cfd43d0c45bdad7f53bc9535e844", - "ddcbf95797e81dd04127885bd001e96695b717e11c52721f6e8ee53f6dea8a6f", - "31ff728f24200f59fa4958e6c26de03d172b320e6eef2b8abecf6f94d01dd4ae", +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ + 40000: testBlock{ + size: 1385, + time: 1327728573, + txs: []string{ + "e193a821393b20b99f4a4e05a481368ef8a8cfd43d0c45bdad7f53bc9535e844", + "ddcbf95797e81dd04127885bd001e96695b717e11c52721f6e8ee53f6dea8a6f", + "31ff728f24200f59fa4958e6c26de03d172b320e6eef2b8abecf6f94d01dd4ae", + }, }, } @@ -84,7 +94,7 @@ func helperLoadBlock(t *testing.T, height int) []byte { func TestParseBlock(t *testing.T) { p := NewNamecoinParser(GetChainParams("main"), &btc.Configuration{}) - for height, txs := range testParseBlockTxs { + for height, tb := range testParseBlockTxs { b := helperLoadBlock(t, height) blk, err := p.ParseBlock(b) @@ -92,11 +102,19 @@ func TestParseBlock(t *testing.T) { t.Fatal(err) } - if len(blk.Txs) != len(txs) { - t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(txs)) + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) } - for ti, tx := range txs { + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { if blk.Txs[ti].Txid != tx { t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) } diff --git a/bchain/coins/namecoin/namecoinrpc.go b/bchain/coins/namecoin/namecoinrpc.go index faa14fd3..6ab0b0aa 100644 --- a/bchain/coins/namecoin/namecoinrpc.go +++ b/bchain/coins/namecoin/namecoinrpc.go @@ -24,6 +24,7 @@ func NewNamecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false return s, nil } @@ -69,8 +70,3 @@ func (b *NamecoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error } return b.GetBlockWithoutHeader(hash, height) } - -// EstimateSmartFee returns fee estimation. -func (b *NamecoinRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - return b.EstimateFee(blocks) -} diff --git a/bchain/coins/namecoin/namecoinrpc_test.go b/bchain/coins/namecoin/namecoinrpc_test.go deleted file mode 100644 index d50ef581..00000000 --- a/bchain/coins/namecoin/namecoinrpc_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// +build integration - -package namecoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewNamecoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*NamecoinRPC) - cli.Parser = NewNamecoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Namecoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestNamecoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestNamecoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestNamecoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestNamecoinRPC_GetTransactionForMempool(t *testing.T) { - // extra opcodes (name_new, name_firstupdate, name_update) aren't supported, so some transactions - // in mempool can't be parsed correctly - t.Skipf("Skipped because of instability") -} - -func TestNamecoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestNamecoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestNamecoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestNamecoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} - -func TestNamecoinRPC_GetBestBlockHash(t *testing.T) { - rpcTest.TestGetBestBlockHash(t) -} - -func TestNamecoinRPC_GetBestBlockHeight(t *testing.T) { - rpcTest.TestGetBestBlockHeight(t) -} - -func TestNamecoinRPC_GetBlockHeader(t *testing.T) { - rpcTest.TestGetBlockHeader(t) -} diff --git a/bchain/coins/utils/address.go b/bchain/coins/utils/address.go deleted file mode 100644 index 9fb2708e..00000000 --- a/bchain/coins/utils/address.go +++ /dev/null @@ -1,42 +0,0 @@ -package utils - -import ( - "crypto/sha256" - "errors" - - "github.com/btcsuite/btcutil/base58" -) - -var ( - // ErrChecksumMismatch describes an error where decoding failed due - // to a bad checksum. - ErrChecksumMismatch = errors.New("checksum mismatch") - - // ErrInvalidFormat describes an error where decoding failed due to invalid version - ErrInvalidFormat = errors.New("invalid format: version and/or checksum bytes missing") -) - -// checksum: first four bytes of sha256^2 -func checksum(input []byte) (cksum [4]byte) { - h := sha256.Sum256(input) - h2 := sha256.Sum256(h[:]) - copy(cksum[:], h2[:4]) - return -} - -// CheckDecode decodes a string that was encoded with CheckEncode and verifies the checksum. -func CheckDecode(input string) (result []byte, version []byte, err error) { - decoded := base58.Decode(input) - if len(decoded) < 5 { - return nil, nil, ErrInvalidFormat - } - version = append(version, decoded[0:2]...) - var cksum [4]byte - copy(cksum[:], decoded[len(decoded)-4:]) - if checksum(decoded[:len(decoded)-4]) != cksum { - return nil, nil, ErrChecksumMismatch - } - payload := decoded[2 : len(decoded)-4] - result = append(result, payload...) - return -} diff --git a/bchain/coins/utils/parserutils.go b/bchain/coins/utils/parserutils.go index 4fbe422c..c59979ed 100644 --- a/bchain/coins/utils/parserutils.go +++ b/bchain/coins/utils/parserutils.go @@ -33,7 +33,7 @@ func DecodeTransactions(r io.Reader, pver uint32, enc wire.MessageEncoding, blk if txCount > maxTxPerBlock { str := fmt.Sprintf("too many transactions to fit into a block "+ "[count %d, max %d]", txCount, maxTxPerBlock) - return &wire.MessageError{Func: "btg.decodeTransactions", Description: str} + return &wire.MessageError{Func: "utils.decodeTransactions", Description: str} } blk.Transactions = make([]*wire.MsgTx, 0, txCount) diff --git a/bchain/coins/vertcoin/vertcoinparser.go b/bchain/coins/vertcoin/vertcoinparser.go index 6f33a9ae..ef7d82fc 100644 --- a/bchain/coins/vertcoin/vertcoinparser.go +++ b/bchain/coins/vertcoin/vertcoinparser.go @@ -3,8 +3,8 @@ package vertcoin import ( "blockbook/bchain/coins/btc" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -18,17 +18,17 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic - MainNetParams.PubKeyHashAddrID = 71 - MainNetParams.ScriptHashAddrID = 5 + MainNetParams.PubKeyHashAddrID = []byte{71} + MainNetParams.ScriptHashAddrID = []byte{5} MainNetParams.Bech32HRPSegwit = "vtc" TestNetParams = chaincfg.TestNet3Params TestNetParams.Net = TestnetMagic - TestNetParams.PubKeyHashAddrID = 74 - TestNetParams.ScriptHashAddrID = 196 + TestNetParams.PubKeyHashAddrID = []byte{74} + TestNetParams.ScriptHashAddrID = []byte{196} TestNetParams.Bech32HRPSegwit = "tvtc" err := chaincfg.Register(&MainNetParams) @@ -53,6 +53,9 @@ func NewVertcoinParser(params *chaincfg.Params, c *btc.Configuration) *VertcoinP // GetChainParams contains network parameters for the main Vertcoin network, // and the test Vertcoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/vertcoin/vertcoinparser_test.go b/bchain/coins/vertcoin/vertcoinparser_test.go index 558ae51f..5cdcb5c6 100644 --- a/bchain/coins/vertcoin/vertcoinparser_test.go +++ b/bchain/coins/vertcoin/vertcoinparser_test.go @@ -6,11 +6,12 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/hex" + "math/big" "reflect" "testing" ) -func TestAddressToOutputScript_Mainnet(t *testing.T) { +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { type args struct { address string } @@ -55,14 +56,14 @@ func TestAddressToOutputScript_Mainnet(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parser.AddressToOutputScript(tt.args.address) + got, err := parser.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { - t.Errorf("AddressToOutputScript() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) return } h := hex.EncodeToString(got) if !reflect.DeepEqual(h, tt.want) { - t.Errorf("AddressToOutputScript() = %v, want %v", h, tt.want) + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) } }) } @@ -75,18 +76,6 @@ var ( ) func init() { - var ( - addr1, addr2 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("Vp1UqzsmVecaexfbWFGSFFL5x1g2XQnrGR") - if err == nil { - addr2, err = bchain.NewBaseAddress("38A1RNvbA5c9wNRfyLVn1FCH5TPKJVG8YR") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "010000000146fd781834a34e0399ccda1edf9ec47d715e17d904ad0958d533a240b3605ad6000000006a473044022026b352a0c35c232342339e2b50ec9f04587b990d5213174e368cc76dc82686f002207d0787461ad846825872a50d3d6fc748d5a836575c1daf6ad0ca602f9c4a8826012103d36b6b829c571ed7caa565eca9bdc2aa36519b7ab8551ace5edb0356d477ad3cfdffffff020882a400000000001976a91499b16da88a7e29b913b6131df2644d6d06cb331b88ac80f0fa020000000017a91446eb90e002f137f05385896c882fe000cc2e967f8774870e00", Blocktime: 1529925180, @@ -105,26 +94,24 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 0.10781192, - N: 0, + ValueSat: *big.NewInt(10781192), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a91499b16da88a7e29b913b6131df2644d6d06cb331b88ac", Addresses: []string{ "Vp1UqzsmVecaexfbWFGSFFL5x1g2XQnrGR", }, }, - Address: addr1, }, { - Value: 0.5000000, - N: 1, + ValueSat: *big.NewInt(50000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "a91446eb90e002f137f05385896c882fe000cc2e967f87", Addresses: []string{ "38A1RNvbA5c9wNRfyLVn1FCH5TPKJVG8YR", }, }, - Address: addr2, }, }, } diff --git a/bchain/coins/vertcoin/vertcoinrpc.go b/bchain/coins/vertcoin/vertcoinrpc.go index 878b9682..f282dc78 100644 --- a/bchain/coins/vertcoin/vertcoinrpc.go +++ b/bchain/coins/vertcoin/vertcoinrpc.go @@ -24,6 +24,7 @@ func NewVertcoinRPC(config json.RawMessage, pushHandler func(bchain.Notification b.(*btc.BitcoinRPC), } s.RPCMarshaler = btc.JSONMarshalerV2{} + s.ChainConfig.SupportsEstimateFee = false return s, nil } @@ -54,8 +55,3 @@ func (b *VertcoinRPC) Initialize() error { return nil } - -// EstimateFee returns fee estimation. -func (b *VertcoinRPC) EstimateFee(blocks int) (float64, error) { - return b.EstimateSmartFee(blocks, true) -} diff --git a/bchain/coins/vertcoin/vertcoinrpc_test.go b/bchain/coins/vertcoin/vertcoinrpc_test.go deleted file mode 100644 index aa4c615e..00000000 --- a/bchain/coins/vertcoin/vertcoinrpc_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// +build integration - -package vertcoin - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewVertcoinRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*VertcoinRPC) - cli.Parser = NewVertcoinParser(GetChainParams("main"), cli.ChainConfig) - if err != nil { - return nil, err - } - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var rpcTest *rpc.Test - -func TestMain(m *testing.M) { - flag.Parse() - t, err := rpc.NewTest("Vertcoin", getRPCClient) - if err != nil { - panic(err) - } - - rpcTest = t - - os.Exit(m.Run()) -} - -func TestVertcoinRPC_GetBlockHash(t *testing.T) { - rpcTest.TestGetBlockHash(t) -} - -func TestVertcoinRPC_GetBlock(t *testing.T) { - rpcTest.TestGetBlock(t) -} - -func TestVertcoinRPC_GetTransaction(t *testing.T) { - rpcTest.TestGetTransaction(t) -} - -func TestVertcoinRPC_GetTransactionForMempool(t *testing.T) { - rpcTest.TestGetTransactionForMempool(t) -} - -func TestVertcoinRPC_MempoolSync(t *testing.T) { - rpcTest.TestMempoolSync(t) -} - -func TestVertcoinRPC_GetMempoolEntry(t *testing.T) { - rpcTest.TestGetMempoolEntry(t) -} - -func TestVertcoinRPC_EstimateSmartFee(t *testing.T) { - rpcTest.TestEstimateSmartFee(t) -} - -func TestVertcoinRPC_EstimateFee(t *testing.T) { - rpcTest.TestEstimateFee(t) -} - -func TestVertcoinRPC_GetBestBlockHash(t *testing.T) { - rpcTest.TestGetBestBlockHash(t) -} - -func TestVertcoinRPC_GetBestBlockHeight(t *testing.T) { - rpcTest.TestGetBestBlockHeight(t) -} - -func TestVertcoinRPC_GetBlockHeader(t *testing.T) { - rpcTest.TestGetBlockHeader(t) -} diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index eacf8bbd..ea4917c3 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -3,10 +3,9 @@ package zec import ( "blockbook/bchain" "blockbook/bchain/coins/btc" - "blockbook/bchain/coins/utils" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" ) const ( @@ -15,18 +14,48 @@ const ( RegtestMagic wire.BitcoinNet = 0x5f3fe8aa ) -// ZCashParser handle -type ZCashParser struct { - *bchain.BaseParser +var ( + MainNetParams chaincfg.Params + TestNetParams chaincfg.Params +) + +func initParams() { + MainNetParams = chaincfg.MainNetParams + MainNetParams.Net = MainnetMagic + + // Address encoding magics + MainNetParams.AddressMagicLen = 2 + MainNetParams.PubKeyHashAddrID = []byte{0x1C, 0xB8} // base58 prefix: t1 + MainNetParams.ScriptHashAddrID = []byte{0x1C, 0xBD} // base58 prefix: t3 + + TestNetParams = chaincfg.TestNet3Params + TestNetParams.Net = TestnetMagic + + // Address encoding magics + TestNetParams.AddressMagicLen = 2 + TestNetParams.PubKeyHashAddrID = []byte{0x1D, 0x25} // base58 prefix: tm + TestNetParams.ScriptHashAddrID = []byte{0x1C, 0xBA} // base58 prefix: t2 + + err := chaincfg.Register(&MainNetParams) + if err == nil { + err = chaincfg.Register(&TestNetParams) + } + if err != nil { + panic(err) + } } -// NewZCAshParser returns new ZCAshParser instance -func NewZCashParser(c *btc.Configuration) *ZCashParser { +// ZCashParser handle +type ZCashParser struct { + *btc.BitcoinParser + baseparser *bchain.BaseParser +} + +// NewZCashParser returns new ZCashParser instance +func NewZCashParser(params *chaincfg.Params, c *btc.Configuration) *ZCashParser { return &ZCashParser{ - &bchain.BaseParser{ - AddressFactory: bchain.NewBaseAddress, - BlockAddressesToKeep: c.BlockAddressesToKeep, - }, + BitcoinParser: btc.NewBitcoinParser(params, c), + baseparser: &bchain.BaseParser{}, } } @@ -34,33 +63,29 @@ func NewZCashParser(c *btc.Configuration) *ZCashParser { // the regression test ZCash network, the test ZCash network and // the simulation test ZCash network, in this order func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } var params *chaincfg.Params switch chain { case "test": - params = &chaincfg.TestNet3Params - params.Net = TestnetMagic + return &TestNetParams case "regtest": params = &chaincfg.RegressionNetParams params.Net = RegtestMagic default: - params = &chaincfg.MainNetParams - params.Net = MainnetMagic + return &MainNetParams } return params } -// GetAddrIDFromVout returns internal address representation of given transaction output -func (p *ZCashParser) GetAddrIDFromVout(output *bchain.Vout) ([]byte, error) { - if len(output.ScriptPubKey.Addresses) != 1 { - return nil, nil - } - hash, _, err := utils.CheckDecode(output.ScriptPubKey.Addresses[0]) - return hash, err +// PackTx packs transaction to byte array using protobuf +func (p *ZCashParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { + return p.baseparser.PackTx(tx, height, blockTime) } -// GetAddrIDFromAddress returns internal address representation of given address -func (p *ZCashParser) GetAddrIDFromAddress(address string) ([]byte, error) { - hash, _, err := utils.CheckDecode(address) - return hash, err +// UnpackTx unpacks transaction from protobuf byte array +func (p *ZCashParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { + return p.baseparser.UnpackTx(buf) } diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index 0bb61fe1..ebaf5135 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -5,7 +5,9 @@ package zec import ( "blockbook/bchain" "blockbook/bchain/coins/btc" + "bytes" "encoding/hex" + "math/big" "reflect" "testing" ) @@ -13,26 +15,11 @@ import ( var ( testTx1, testTx2 bchain.Tx - testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a4b091e6c90cd3ebc664010001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" - testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a4b09257b2170264d504010001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a4b099a9999999999b93f10011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" + testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a490a05043c1aec8e10001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" + testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a490a050184a7a72310001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a470a0398968010011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" ) func init() { - var ( - addr1, addr2, addr3 bchain.Address - err error - ) - addr1, err = bchain.NewBaseAddress("t1Y4yL14ACHaAbjemkdpW7nYNHWnv1yQbDA") - if err == nil { - addr2, err = bchain.NewBaseAddress("t1VmHTTwpEtwvojxodN2CSQqLYi1hzY3cAq") - } - if err == nil { - addr3, err = bchain.NewBaseAddress("t1ecxMXpphUTRQXGLXnVhJ6ucqD3DZipddg") - } - if err != nil { - panic(err) - } - testTx1 = bchain.Tx{ Hex: "02000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e1802", Blocktime: 1521645728, @@ -51,15 +38,14 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 181.88266638, - N: 0, + ValueSat: *big.NewInt(18188266638), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac", Addresses: []string{ "t1Y4yL14ACHaAbjemkdpW7nYNHWnv1yQbDA", }, }, - Address: addr1, }, }, } @@ -82,31 +68,74 @@ func init() { }, Vout: []bchain.Vout{ { - Value: 65.20547107, - N: 0, + ValueSat: *big.NewInt(6520547107), + N: 0, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914826f87806ddd4643730be99b41c98acc379e83db88ac", Addresses: []string{ "t1VmHTTwpEtwvojxodN2CSQqLYi1hzY3cAq", }, }, - Address: addr2, }, { - Value: .1, - N: 1, + ValueSat: *big.NewInt(10000000), + N: 1, ScriptPubKey: bchain.ScriptPubKey{ Hex: "76a914e395634b7684289285926d4c64db395b783720ec88ac", Addresses: []string{ "t1ecxMXpphUTRQXGLXnVhJ6ucqD3DZipddg", }, }, - Address: addr3, }, }, } } +func TestGetAddrDesc(t *testing.T) { + type args struct { + tx bchain.Tx + parser *ZCashParser + } + tests := []struct { + name string + args args + }{ + { + name: "zec-1", + args: args{ + tx: testTx1, + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), + }, + }, + { + name: "zec-2", + args: args{ + tx: testTx2, + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for n, vout := range tt.args.tx.Vout { + got1, err := tt.args.parser.GetAddrDescFromVout(&vout) + if err != nil { + t.Errorf("getAddrDescFromVout() error = %v, vout = %d", err, n) + return + } + got2, err := tt.args.parser.GetAddrDescFromAddress(vout.ScriptPubKey.Addresses[0]) + if err != nil { + t.Errorf("getAddrDescFromAddress() error = %v, vout = %d", err, n) + return + } + if !bytes.Equal(got1, got2) { + t.Errorf("Address descriptors mismatch: got1 = %v, got2 = %v", got1, got2) + } + } + }) + } +} + func TestPackTx(t *testing.T) { type args struct { tx bchain.Tx @@ -126,7 +155,7 @@ func TestPackTx(t *testing.T) { tx: testTx1, height: 292272, blockTime: 1521645728, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: testTxPacked1, wantErr: false, @@ -137,7 +166,7 @@ func TestPackTx(t *testing.T) { tx: testTx2, height: 292217, blockTime: 1521637604, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: testTxPacked2, wantErr: false, @@ -174,7 +203,7 @@ func TestUnpackTx(t *testing.T) { name: "zec-1", args: args{ packedTx: testTxPacked1, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: &testTx1, want1: 292272, @@ -184,7 +213,7 @@ func TestUnpackTx(t *testing.T) { name: "zec-2", args: args{ packedTx: testTxPacked2, - parser: NewZCashParser(&btc.Configuration{}), + parser: NewZCashParser(GetChainParams("main"), &btc.Configuration{}), }, want: &testTx2, want1: 292217, diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 495fa7af..6c9b023b 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -22,6 +22,7 @@ func NewZCashRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp BitcoinRPC: b.(*btc.BitcoinRPC), } z.RPCMarshaler = btc.JSONMarshalerV1{} + z.ChainConfig.SupportsEstimateSmartFee = false return z, nil } @@ -34,7 +35,7 @@ func (z *ZCashRPC) Initialize() error { params := GetChainParams(chainName) - z.Parser = NewZCashParser(z.ChainConfig) + z.Parser = NewZCashParser(params, z.ChainConfig) // parameters for getInfo request if params.Net == MainnetMagic { @@ -114,14 +115,6 @@ func (z *ZCashRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { return z.GetTransaction(txid) } -// EstimateSmartFee returns fee estimation. -func (z *ZCashRPC) EstimateSmartFee(blocks int, conservative bool) (float64, error) { - glog.V(1).Info("rpc: estimatesmartfee") - - // return z.estimateFee(blocks) - return z.EstimateFee(blocks) -} - // GetMempoolEntry returns mempool data for given transaction func (z *ZCashRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { return nil, errors.New("GetMempoolEntry: not implemented") diff --git a/bchain/coins/zec/zcashrpc_test.go b/bchain/coins/zec/zcashrpc_test.go deleted file mode 100644 index f98520c6..00000000 --- a/bchain/coins/zec/zcashrpc_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// +build integration - -package zec - -import ( - "blockbook/bchain" - "blockbook/bchain/tests/rpc" - "encoding/json" - "flag" - "os" - "testing" -) - -func getRPCClient(cfg json.RawMessage) (bchain.BlockChain, error) { - c, err := NewZCashRPC(cfg, nil) - if err != nil { - return nil, err - } - cli := c.(*ZCashRPC) - cli.Parser = NewZCashParser(cli.ChainConfig) - cli.Mempool = bchain.NewUTXOMempool(cli, cli.ChainConfig.MempoolWorkers, cli.ChainConfig.MempoolSubWorkers) - return cli, nil -} - -var tests struct { - mainnet *rpc.Test - testnet *rpc.Test -} - -func TestMain(m *testing.M) { - flag.Parse() - - t, err := rpc.NewTest("Zcash", getRPCClient) - if err != nil { - panic(err) - } - - tests.mainnet = t - - t, err = rpc.NewTest("Zcash Testnet", getRPCClient) - if err != nil { - panic(err) - } - - tests.testnet = t - - os.Exit(m.Run()) -} - -func TestZCashRPC_GetBlockHash(t *testing.T) { - tests.mainnet.TestGetBlockHash(t) -} - -func TestZCashRPC_GetBlock(t *testing.T) { - tests.mainnet.TestGetBlock(t) -} - -func TestZCashRPC_GetTransaction(t *testing.T) { - tests.mainnet.TestGetTransaction(t) -} - -func TestZCashRPC_GetTransactionForMempool(t *testing.T) { - tests.mainnet.TestGetTransactionForMempool(t) -} - -func TestZCashRPC_MempoolSync(t *testing.T) { - tests.mainnet.TestMempoolSync(t) -} - -func TestZCashRPC_EstimateSmartFee(t *testing.T) { - tests.mainnet.TestEstimateSmartFee(t) -} - -func TestZCashRPC_EstimateFee(t *testing.T) { - tests.mainnet.TestEstimateFee(t) -} - -func TestZCashRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestZCashRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestZCashRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} - -func TestZCashTestnetRPC_GetBlockHash(t *testing.T) { - tests.testnet.TestGetBlockHash(t) -} - -func TestZCashTestnetRPC_GetBlock(t *testing.T) { - tests.testnet.TestGetBlock(t) -} - -func TestZCashTestnetRPC_GetTransaction(t *testing.T) { - tests.testnet.TestGetTransaction(t) -} - -func TestZCashTestnetRPC_GetTransactionForMempool(t *testing.T) { - tests.testnet.TestGetTransactionForMempool(t) -} - -func TestZCashTestnetRPC_MempoolSync(t *testing.T) { - tests.testnet.TestMempoolSync(t) -} - -func TestZCashTestnetRPC_EstimateSmartFee(t *testing.T) { - tests.testnet.TestEstimateSmartFee(t) -} - -func TestZCashTestnetRPC_EstimateFee(t *testing.T) { - tests.testnet.TestEstimateFee(t) -} - -func TestZCashTestnetRPC_GetBestBlockHash(t *testing.T) { - tests.mainnet.TestGetBestBlockHash(t) -} - -func TestZCashTestnetRPC_GetBestBlockHeight(t *testing.T) { - tests.mainnet.TestGetBestBlockHeight(t) -} - -func TestZCashTestnetRPC_GetBlockHeader(t *testing.T) { - tests.mainnet.TestGetBlockHeader(t) -} diff --git a/bchain/mempool_nonutxo.go b/bchain/mempool_nonutxo.go index 23f950c6..28485dab 100644 --- a/bchain/mempool_nonutxo.go +++ b/bchain/mempool_nonutxo.go @@ -12,7 +12,7 @@ type NonUTXOMempool struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrIDToTx map[string][]outpoint + addrDescToTx map[string][]outpoint } // NewNonUTXOMempool creates new mempool handler. @@ -23,13 +23,18 @@ func NewNonUTXOMempool(chain BlockChain) *NonUTXOMempool { // GetTransactions returns slice of mempool transactions for given address func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { parser := m.chain.GetChainParser() - addrID, err := parser.GetAddrIDFromAddress(address) + addrDesc, err := parser.GetAddrDescFromAddress(address) if err != nil { return nil, err } + return m.GetAddrDescTransactions(addrDesc) +} + +// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor +func (m *NonUTXOMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]string, error) { m.mux.Lock() defer m.mux.Unlock() - outpoints := m.addrIDToTx[string(addrID)] + outpoints := m.addrDescToTx[string(addrDesc)] txs := make([]string, 0, len(outpoints)) for _, o := range outpoints { txs = append(txs, o.txid) @@ -37,11 +42,11 @@ func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { return txs, nil } -func (m *NonUTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrIDToTx map[string][]outpoint) { +func (m *NonUTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]outpoint) { m.mux.Lock() defer m.mux.Unlock() m.txToInputOutput = newTxToInputOutput - m.addrIDToTx = newAddrIDToTx + m.addrDescToTx = newAddrDescToTx } // Resync gets mempool transactions and maps outputs to transactions. @@ -57,7 +62,7 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { parser := m.chain.GetChainParser() // allocate slightly larger capacity of the maps newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) - newAddrIDToTx := make(map[string][]outpoint, len(m.addrIDToTx)+5) + newAddrDescToTx := make(map[string][]outpoint, len(m.addrDescToTx)+5) for _, txid := range txs { io, exists := m.txToInputOutput[txid] if !exists { @@ -68,15 +73,15 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { } io = make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) for _, output := range tx.Vout { - addrID, err := parser.GetAddrIDFromVout(&output) + addrDesc, err := parser.GetAddrDescFromVout(&output) if err != nil { if err != ErrAddressMissing { - glog.Error("error in output addrID in ", txid, " ", output.N, ": ", err) + glog.Error("error in output addrDesc in ", txid, " ", output.N, ": ", err) } continue } - if len(addrID) > 0 { - io = append(io, addrIndex{string(addrID), int32(output.N)}) + if len(addrDesc) > 0 { + io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0], true) @@ -85,12 +90,12 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { for _, input := range tx.Vin { for i, a := range input.Addresses { if len(a) > 0 { - addrID, err := parser.GetAddrIDFromAddress(a) + addrDesc, err := parser.GetAddrDescFromAddress(a) if err != nil { - glog.Error("error in input addrID in ", txid, " ", a, ": ", err) + glog.Error("error in input addrDesc in ", txid, " ", a, ": ", err) continue } - io = append(io, addrIndex{string(addrID), int32(^i)}) + io = append(io, addrIndex{string(addrDesc), int32(^i)}) if onNewTxAddr != nil { onNewTxAddr(tx.Txid, a, false) } @@ -100,10 +105,10 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { } newTxToInputOutput[txid] = io for _, si := range io { - newAddrIDToTx[si.addrID] = append(newAddrIDToTx[si.addrID], outpoint{txid, si.n}) + newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], outpoint{txid, si.n}) } } - m.updateMappings(newTxToInputOutput, newAddrIDToTx) + m.updateMappings(newTxToInputOutput, newAddrDescToTx) glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txToInputOutput), " transactions in mempool") return len(m.txToInputOutput), nil } diff --git a/bchain/mempool_utxo.go b/bchain/mempool_utxo.go index 5483ca8e..54017e00 100644 --- a/bchain/mempool_utxo.go +++ b/bchain/mempool_utxo.go @@ -9,8 +9,8 @@ import ( // addrIndex and outpoint are used also in non utxo mempool type addrIndex struct { - addrID string - n int32 + addrDesc string + n int32 } type outpoint struct { @@ -28,7 +28,7 @@ type UTXOMempool struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrIDToTx map[string][]outpoint + addrDescToTx map[string][]outpoint chanTxid chan string chanAddrIndex chan txidio onNewTxAddr OnNewTxAddrFunc @@ -70,13 +70,18 @@ func NewUTXOMempool(chain BlockChain, workers int, subworkers int) *UTXOMempool // GetTransactions returns slice of mempool transactions for given address func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { parser := m.chain.GetChainParser() - addrID, err := parser.GetAddrIDFromAddress(address) + addrDesc, err := parser.GetAddrDescFromAddress(address) if err != nil { return nil, err } + return m.GetAddrDescTransactions(addrDesc) +} + +// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor +func (m *UTXOMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]string, error) { m.mux.Lock() defer m.mux.Unlock() - outpoints := m.addrIDToTx[string(addrID)] + outpoints := m.addrDescToTx[string(addrDesc)] txs := make([]string, 0, len(outpoints)) for _, o := range outpoints { txs = append(txs, o.txid) @@ -84,15 +89,14 @@ func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { return txs, nil } -func (m *UTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrIDToTx map[string][]outpoint) { +func (m *UTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]outpoint) { m.mux.Lock() defer m.mux.Unlock() m.txToInputOutput = newTxToInputOutput - m.addrIDToTx = newAddrIDToTx + m.addrDescToTx = newAddrDescToTx } func (m *UTXOMempool) getInputAddress(input outpoint) *addrIndex { - // TODO - possibly get from DB unspenttxs - however some output txs can be also in mempool itx, err := m.chain.GetTransactionForMempool(input.txid) if err != nil { glog.Error("cannot get transaction ", input.txid, ": ", err) @@ -102,12 +106,12 @@ func (m *UTXOMempool) getInputAddress(input outpoint) *addrIndex { glog.Error("Vout len in transaction ", input.txid, " ", len(itx.Vout), " input.Vout=", input.vout) return nil } - addrID, err := m.chain.GetChainParser().GetAddrIDFromVout(&itx.Vout[input.vout]) + addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&itx.Vout[input.vout]) if err != nil { - glog.Error("error in addrID in ", input.txid, " ", input.vout, ": ", err) + glog.Error("error in addrDesc in ", input.txid, " ", input.vout, ": ", err) return nil } - return &addrIndex{string(addrID), ^input.vout} + return &addrIndex{string(addrDesc), ^input.vout} } @@ -120,13 +124,13 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul glog.V(2).Info("mempool: gettxaddrs ", txid, ", ", len(tx.Vin), " inputs") io := make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) for _, output := range tx.Vout { - addrID, err := m.chain.GetChainParser().GetAddrIDFromVout(&output) + addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&output) if err != nil { - glog.Error("error in addrID in ", txid, " ", output.N, ": ", err) + glog.Error("error in addrDesc in ", txid, " ", output.N, ": ", err) continue } - if len(addrID) > 0 { - io = append(io, addrIndex{string(addrID), int32(output.N)}) + if len(addrDesc) > 0 { + io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if m.onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { m.onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0], true) @@ -177,13 +181,13 @@ func (m *UTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { glog.V(2).Info("mempool: resync ", len(txs), " txs") // allocate slightly larger capacity of the maps newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) - newAddrIDToTx := make(map[string][]outpoint, len(m.addrIDToTx)+5) + newAddrDescToTx := make(map[string][]outpoint, len(m.addrDescToTx)+5) dispatched := 0 onNewData := func(txid string, io []addrIndex) { if len(io) > 0 { newTxToInputOutput[txid] = io for _, si := range io { - newAddrIDToTx[si.addrID] = append(newAddrIDToTx[si.addrID], outpoint{txid, si.n}) + newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], outpoint{txid, si.n}) } } } @@ -212,7 +216,7 @@ func (m *UTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { tio := <-m.chanAddrIndex onNewData(tio.txid, tio.io) } - m.updateMappings(newTxToInputOutput, newAddrIDToTx) + m.updateMappings(newTxToInputOutput, newAddrDescToTx) m.onNewTxAddr = nil glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txToInputOutput), " transactions in mempool") return len(m.txToInputOutput), nil diff --git a/bchain/tests/rpc/config.json b/bchain/tests/rpc/config.json deleted file mode 100644 index e4019e07..00000000 --- a/bchain/tests/rpc/config.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "Bcash": { - "url": "http://localhost:8031", - "user": "rpc", - "pass": "rpc" - }, - "Bitcoin": { - "url": "http://localhost:8030", - "user": "rpc", - "pass": "rpc" - }, - "Dash": { - "url": "http://localhost:8033", - "user": "rpc", - "pass": "rpc" - }, - "Ethereum": { - "url": "ws://localhost:8036", - "user": null, - "pass": null - }, - "Zcash": { - "url": "http://localhost:8032", - "user": "rpc", - "pass": "rpc" - }, - "Vertcoin": { - "url": "http://localhost:8040", - "user": "rpc", - "pass": "rpc" - }, - "Namecoin": { - "url": "http://localhost:8039", - "user": "rpc", - "pass": "rpc" - }, - "Bcash Testnet": { - "url": "http://localhost:18031", - "user": "rpc", - "pass": "rpc" - }, - "Bitcoin Testnet": { - "url": "http://localhost:18030", - "user": "rpc", - "pass": "rpc" - }, - "Dash Testnet": { - "url": "http://localhost:18033", - "user": "rpc", - "pass": "rpc" - }, - "Ethereum Testnet": { - "url": "ws://localhost:18036", - "user": null, - "pass": null - }, - "Zcash Testnet": { - "url": "http://localhost:18032", - "user": "rpc", - "pass": "rpc" - } -} diff --git a/bchain/tests/rpc/data.go b/bchain/tests/rpc/data.go index 7d50460a..dbe8c1f8 100644 --- a/bchain/tests/rpc/data.go +++ b/bchain/tests/rpc/data.go @@ -1,16 +1,22 @@ -// +build integration - package rpc import ( + "blockbook/bchain" "encoding/json" "errors" - "fmt" "io/ioutil" "path/filepath" "strings" ) +type TestData struct { + BlockHeight uint32 `json:"blockHeight"` + BlockHash string `json:"blockHash"` + BlockTime int64 `json:"blockTime"` + BlockTxs []string `json:"blockTxs"` + TxDetails map[string]*bchain.Tx `json:"txDetails"` +} + func joinPathsWithCommonElement(p1, p2 string) (string, bool) { idx := strings.IndexRune(p2, filepath.Separator) if idx <= 0 { @@ -43,47 +49,7 @@ func readDataFile(dir, relDir, filename string) ([]byte, error) { return ioutil.ReadFile(path) } -var testConfigRegistry map[string]*TestConfig - -func LoadTestConfig(coin string) (*TestConfig, error) { - if testConfigRegistry == nil { - b, err := readDataFile(".", "bchain/tests/rpc", "config.json") - if err != nil { - return nil, err - } - var v map[string]*TestConfig - err = json.Unmarshal(b, &v) - if err != nil { - return nil, err - } - testConfigRegistry = v - } - c, found := testConfigRegistry[coin] - if !found { - return nil, errors.New("Test config not found") - } - return c, nil -} - -func LoadRPCConfig(coin string) (json.RawMessage, error) { - t := `{ - "coin_name": "%s", - "rpcURL": "%s", - "rpcUser": "%s", - "rpcPass": "%s", - "rpcTimeout": 25, - "parse": true - }` - - c, err := LoadTestConfig(coin) - if err != nil { - return json.RawMessage{}, err - } - - return json.RawMessage(fmt.Sprintf(t, coin, c.URL, c.User, c.Pass)), nil -} - -func LoadTestData(coin string) (*TestData, error) { +func LoadTestData(coin string, parser bchain.BlockChainParser) (*TestData, error) { b, err := readDataFile(".", "bchain/tests/rpc/testdata", coin+".json") if err != nil { return nil, err @@ -93,5 +59,16 @@ func LoadTestData(coin string) (*TestData, error) { if err != nil { return nil, err } + // convert amounts in test json to bit.Int and clear the temporary JsonValue + for _, tx := range v.TxDetails { + for i := range tx.Vout { + vout := &tx.Vout[i] + vout.ValueSat, err = parser.AmountToBigInt(vout.JsonValue) + if err != nil { + return nil, err + } + vout.JsonValue = "" + } + } return &v, nil } diff --git a/bchain/tests/rpc/rpc.go b/bchain/tests/rpc/rpc.go deleted file mode 100644 index 4386baa1..00000000 --- a/bchain/tests/rpc/rpc.go +++ /dev/null @@ -1,448 +0,0 @@ -// +build integration - -package rpc - -import ( - "blockbook/bchain" - "encoding/json" - "math/rand" - "net" - "reflect" - "testing" - - "github.com/deckarep/golang-set" -) - -type TestConfig struct { - URL string `json:"url"` - User string `json:"user"` - Pass string `json:"pass"` -} - -type TestData struct { - BlockHeight uint32 `json:"blockHeight"` - BlockHash string `json:"blockHash"` - BlockTxs []string `json:"blockTxs"` - TxDetails map[string]*bchain.Tx `json:"txDetails"` -} - -type Test struct { - Client bchain.BlockChain - TestData *TestData - connected bool -} - -type TestChainFactoryFunc func(json.RawMessage) (bchain.BlockChain, error) - -func NewTest(coin string, factory TestChainFactoryFunc) (*Test, error) { - var ( - connected = true - cli bchain.BlockChain - cfg json.RawMessage - td *TestData - err error - ) - - cfg, err = LoadRPCConfig(coin) - if err != nil { - return nil, err - } - - cli, err = factory(cfg) - if err != nil { - if isNetError(err) { - connected = false - } else { - return nil, err - } - } else { - td, err = LoadTestData(coin) - if err != nil { - return nil, err - } - - if td.TxDetails != nil { - parser := cli.GetChainParser() - - for _, tx := range td.TxDetails { - err := setTxAddresses(parser, tx) - if err != nil { - return nil, err - } - } - } - - _, err = cli.GetBlockChainInfo() - if err != nil && isNetError(err) { - connected = false - } - } - - return &Test{Client: cli, TestData: td, connected: connected}, nil -} - -func isNetError(err error) bool { - if _, ok := err.(net.Error); ok { - return true - } - return false -} - -func setTxAddresses(parser bchain.BlockChainParser, tx *bchain.Tx) error { - // pack and unpack transaction in order to get addresses decoded - ugly but works - var tmp *bchain.Tx - b, err := parser.PackTx(tx, 0, 0) - if err == nil { - tmp, _, err = parser.UnpackTx(b) - if err == nil { - for i := 0; i < len(tx.Vout); i++ { - tx.Vout[i].ScriptPubKey.Addresses = tmp.Vout[i].ScriptPubKey.Addresses - tx.Vout[i].Address = tmp.Vout[i].Address - } - } - } - return err -} - -func (rt *Test) skipUnconnected(t *testing.T) { - if !rt.connected { - t.Skip("Skipping test, not connected to backend service") - } -} - -func (rt *Test) TestGetBlockHash(t *testing.T) { - rt.skipUnconnected(t) - - hash, err := rt.Client.GetBlockHash(rt.TestData.BlockHeight) - if err != nil { - t.Error(err) - return - } - - if hash != rt.TestData.BlockHash { - t.Errorf("GetBlockHash() got %q, want %q", hash, rt.TestData.BlockHash) - } -} - -func (rt *Test) TestGetBlock(t *testing.T) { - rt.skipUnconnected(t) - - blk, err := rt.Client.GetBlock(rt.TestData.BlockHash, 0) - if err != nil { - t.Error(err) - return - } - - if len(blk.Txs) != len(rt.TestData.BlockTxs) { - t.Errorf("GetBlock() number of transactions: got %d, want %d", len(blk.Txs), len(rt.TestData.BlockTxs)) - } - - for ti, tx := range blk.Txs { - if tx.Txid != rt.TestData.BlockTxs[ti] { - t.Errorf("GetBlock() transaction %d: got %s, want %s", ti, tx.Txid, rt.TestData.BlockTxs[ti]) - } - } - -} - -func (rt *Test) TestGetTransaction(t *testing.T) { - rt.skipUnconnected(t) - - for txid, want := range rt.TestData.TxDetails { - got, err := rt.Client.GetTransaction(txid) - if err != nil { - t.Error(err) - return - } - // Confirmations is variable field, we just check if is set and reset it - if got.Confirmations <= 0 { - t.Errorf("GetTransaction() got struct with invalid Confirmations field") - continue - } - got.Confirmations = 0 - - if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransaction() got %v, want %v", got, want) - } - } -} - -func (rt *Test) TestGetTransactionForMempool(t *testing.T) { - rt.skipUnconnected(t) - - for txid, want := range rt.TestData.TxDetails { - // reset fields that are not parsed by BlockChainParser - want.Confirmations, want.Blocktime, want.Time = 0, 0, 0 - - got, err := rt.Client.GetTransactionForMempool(txid) - if err != nil { - t.Fatal(err) - } - // transactions parsed from JSON may contain additional data - got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 - if !reflect.DeepEqual(got, want) { - t.Errorf("GetTransactionForMempool() got %v, want %v", got, want) - } - } -} - -func (rt *Test) getMempool(t *testing.T) []string { - txs, err := rt.Client.GetMempool() - if err != nil { - t.Fatal(err) - } - if len(txs) == 0 { - t.Skip("Skipping test, mempool is empty") - } - - return txs -} - -func (rt *Test) getMempoolAddresses(t *testing.T, txs []string) map[string][]string { - txid2addrs := map[string][]string{} - for i := 0; i < len(txs); i++ { - tx, err := rt.Client.GetTransactionForMempool(txs[i]) - if err != nil { - t.Fatal(err) - } - addrs := []string{} - for _, vin := range tx.Vin { - for _, a := range vin.Addresses { - addrs = append(addrs, a) - } - } - for _, vout := range tx.Vout { - for _, a := range vout.ScriptPubKey.Addresses { - addrs = append(addrs, a) - } - } - if len(addrs) > 0 { - txid2addrs[tx.Txid] = addrs - } - } - return txid2addrs -} - -func (rt *Test) TestMempoolSync(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - txs := rt.getMempool(t) - - n, err := rt.Client.ResyncMempool(nil) - if err != nil { - t.Fatal(err) - } - if n == 0 { - // no transactions to test - continue - } - - txs = intersect(txs, rt.getMempool(t)) - if len(txs) == 0 { - // no transactions to test - continue - } - - txid2addrs := rt.getMempoolAddresses(t, txs) - if len(txid2addrs) == 0 { - t.Skip("Skipping test, no addresses in mempool") - } - - for txid, addrs := range txid2addrs { - for _, a := range addrs { - got, err := rt.Client.GetMempoolTransactions(a) - if err != nil { - t.Fatal(err) - } - if !containsString(got, txid) { - t.Errorf("ResyncMempool() - for address %s, transaction %s wasn't found in mempool", a, txid) - return - } - } - } - - // done - return - } - t.Skip("Skipping test, all attempts to sync mempool failed due to network state changes") -} - -func intersect(a, b []string) []string { - setA := mapset.NewSet() - for _, v := range a { - setA.Add(v) - } - setB := mapset.NewSet() - for _, v := range b { - setB.Add(v) - } - inter := setA.Intersect(setB) - res := make([]string, 0, inter.Cardinality()) - for v := range inter.Iter() { - res = append(res, v.(string)) - } - return res -} - -func containsString(slice []string, s string) bool { - for i := 0; i < len(slice); i++ { - if slice[i] == s { - return true - } - } - return false -} - -func (rt *Test) TestGetMempoolEntry(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - txs := rt.getMempool(t) - h, err := rt.Client.GetBestBlockHeight() - if err != nil { - t.Fatal(err) - } - - txid := txs[rand.Intn(len(txs))] - tx, err := rt.Client.GetTransactionForMempool(txid) - if err != nil { - t.Fatal(err) - } - if tx.Confirmations > 0 { - // tx confirmed - continue - } - - e, err := rt.Client.GetMempoolEntry(txid) - if err != nil { - if err, ok := err.(*bchain.RPCError); ok && err.Code == -5 { - // tx confirmed - continue - } - t.Fatal(err) - } - - if d := int(e.Height) - int(h); d < -1 || d > 1 { - t.Errorf("GetMempoolEntry() got height %d, want %d", e.Height, h) - } - if e.Size <= 0 { - t.Errorf("GetMempoolEntry() got zero or negative size %d", e.Size) - } - if e.Fee <= 0 { - t.Errorf("GetMempoolEntry() got zero or negative fee %f", e.Fee) - } - - // done - return - } - t.Skip("Skipping test, all attempts to get mempool entry failed due to network state changes") -} - -func (rt *Test) TestEstimateSmartFee(t *testing.T) { - rt.skipUnconnected(t) - - for _, blocks := range []int{1, 2, 3, 5, 10} { - fee, err := rt.Client.EstimateSmartFee(blocks, true) - if err != nil { - t.Error(err) - } - if fee != -1 && fee < 0 { - t.Errorf("EstimateSmartFee() returned unexpected fee rate: %f", fee) - } - } -} - -func (rt *Test) TestEstimateFee(t *testing.T) { - rt.skipUnconnected(t) - - for _, blocks := range []int{1, 2, 3, 5, 10} { - fee, err := rt.Client.EstimateFee(blocks) - if err != nil { - t.Error(err) - } - if fee != -1 && fee < 0 { - t.Errorf("EstimateFee() returned unexpected fee rate: %f", fee) - } - } -} - -func (rt *Test) TestGetBestBlockHash(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - hash, err := rt.Client.GetBestBlockHash() - if err != nil { - t.Fatal(err) - } - - height, err := rt.Client.GetBestBlockHeight() - if err != nil { - t.Fatal(err) - } - hh, err := rt.Client.GetBlockHash(height) - if err != nil { - t.Fatal(err) - } - if hash != hh { - continue - } - - // we expect no next block - _, err = rt.Client.GetBlock("", height+1) - if err != nil { - if err != bchain.ErrBlockNotFound { - t.Error(err) - } - return - } - } - t.Error("GetBestBlockHash() didn't get the best hash") -} - -func (rt *Test) TestGetBestBlockHeight(t *testing.T) { - rt.skipUnconnected(t) - - for i := 0; i < 3; i++ { - height, err := rt.Client.GetBestBlockHeight() - if err != nil { - t.Fatal(err) - } - - // we expect no next block - _, err = rt.Client.GetBlock("", height+1) - if err != nil { - if err != bchain.ErrBlockNotFound { - t.Error(err) - } - return - } - } - t.Error("GetBestBlockHeigh() didn't get the the best heigh") -} - -func (rt *Test) TestGetBlockHeader(t *testing.T) { - rt.skipUnconnected(t) - - want := &bchain.BlockHeader{ - Hash: rt.TestData.BlockHash, - Height: rt.TestData.BlockHeight, - } - - got, err := rt.Client.GetBlockHeader(rt.TestData.BlockHash) - if err != nil { - t.Fatal(err) - } - - // Confirmations is variable field, we just check if is set and reset it - if got.Confirmations <= 0 { - t.Fatalf("GetBlockHeader() got struct with invalid Confirmations field") - } - got.Confirmations = 0 - - got.Prev, got.Next = "", "" - - if !reflect.DeepEqual(got, want) { - t.Errorf("GetBlockHeader() got=%v, want=%v", got, want) - } -} diff --git a/bchain/tests/rpc/rpc_test.go b/bchain/tests/rpc/rpc_test.go new file mode 100644 index 00000000..b5658846 --- /dev/null +++ b/bchain/tests/rpc/rpc_test.go @@ -0,0 +1,492 @@ +// +build integration + +package rpc + +import ( + "blockbook/bchain" + "blockbook/bchain/coins" + "blockbook/build/tools" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/deckarep/golang-set" +) + +var testMap = map[string]func(t *testing.T, th *TestHandler){ + "GetBlockHash": testGetBlockHash, + "GetBlock": testGetBlock, + "GetTransaction": testGetTransaction, + "GetTransactionForMempool": testGetTransactionForMempool, + "MempoolSync": testMempoolSync, + "EstimateSmartFee": testEstimateSmartFee, + "EstimateFee": testEstimateFee, + "GetBestBlockHash": testGetBestBlockHash, + "GetBestBlockHeight": testGetBestBlockHeight, + "GetBlockHeader": testGetBlockHeader, +} + +type TestHandler struct { + Client bchain.BlockChain + TestData *TestData + connected bool +} + +var notConnectedError = errors.New("Not connected to backend server") + +func TestRPCIntegration(t *testing.T) { + src := os.Getenv("BLOCKBOOK_SRC") + if src == "" { + t.Fatalf("Missing environment variable BLOCKBOOK_SRC") + } + + configsDir := filepath.Join(src, "configs") + templateDir := filepath.Join(src, "build/templates") + + noTests := 0 + skippedTests := make([]string, 0, 10) + + err := filepath.Walk(filepath.Join(configsDir, "coins"), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Name()[0] == '.' { + return nil + } + + n := strings.TrimSuffix(info.Name(), ".json") + c, err := build.LoadConfig(configsDir, n) + if err != nil { + t.Errorf("%s: cannot load configuration: %s", n, err) + return nil + } + if len(c.IntegrationTests["rpc"]) == 0 { + return nil + } + + cfg, err := makeBlockChainConfig(c, templateDir) + if err != nil { + t.Errorf("%s: cannot make blockchain config: %s", n, err) + return nil + } + + t.Run(c.Coin.Alias, func(t *testing.T) { + noTests += 1 + err := runTests(t, c.Coin.Name, c.Coin.Alias, cfg, c.IntegrationTests["rpc"]) + if err != nil { + if err == notConnectedError { + skippedTests = append(skippedTests, c.Coin.Alias) + t.Skip(err) + } + t.Fatal(err) + } + }) + + return nil + }) + + if err != nil { + t.Fatal(err) + } + + if len(skippedTests) > 0 { + t.Errorf("Too many skipped tests due to connection issues: %q", skippedTests) + } +} + +func makeBlockChainConfig(c *build.Config, templateDir string) (json.RawMessage, error) { + outputDir, err := ioutil.TempDir("", "rpc_test") + if err != nil { + return nil, err + } + defer os.RemoveAll(outputDir) + + err = build.GeneratePackageDefinitions(c, templateDir, outputDir) + if err != nil { + return nil, err + } + + b, err := ioutil.ReadFile(filepath.Join(outputDir, "blockbook", "blockchaincfg.json")) + if err != nil { + return nil, err + } + + var v json.RawMessage + err = json.Unmarshal(b, &v) + if err != nil { + return nil, err + } + + return v, nil +} + +func runTests(t *testing.T, coinName, coinAlias string, cfg json.RawMessage, tests []string) error { + cli, err := initBlockChain(coinName, cfg) + if err != nil { + if err == notConnectedError { + return err + } + t.Fatal(err) + } + td, err := LoadTestData(coinAlias, cli.GetChainParser()) + if err != nil { + t.Fatalf("Test data loading failed: %s", err) + } + + if td.TxDetails != nil { + parser := cli.GetChainParser() + + for _, tx := range td.TxDetails { + err := setTxAddresses(parser, tx) + if err != nil { + t.Fatalf("Test data loading failed: %s", err) + } + } + } + + h := TestHandler{Client: cli, TestData: td} + + for _, test := range tests { + if f, found := testMap[test]; found { + t.Run(test, func(t *testing.T) { f(t, &h) }) + } else { + t.Errorf("%s: test not found", test) + continue + } + } + + return nil +} + +func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, error) { + factory, found := coins.BlockChainFactories[coinName] + if !found { + return nil, fmt.Errorf("Factory function not found") + } + + cli, err := factory(cfg, func(_ bchain.NotificationType) {}) + if err != nil { + if isNetError(err) { + return nil, notConnectedError + } + return nil, fmt.Errorf("Factory function failed: %s", err) + } + + err = cli.Initialize() + if err != nil { + if isNetError(err) { + return nil, notConnectedError + } + return nil, fmt.Errorf("BlockChain initialization failed: %s", err) + } + + return cli, nil +} + +func isNetError(err error) bool { + if _, ok := err.(net.Error); ok { + return true + } + return false +} + +func setTxAddresses(parser bchain.BlockChainParser, tx *bchain.Tx) error { + // pack and unpack transaction in order to get addresses decoded - ugly but works + var tmp *bchain.Tx + b, err := parser.PackTx(tx, 0, 0) + if err == nil { + tmp, _, err = parser.UnpackTx(b) + if err == nil { + for i := 0; i < len(tx.Vout); i++ { + tx.Vout[i].ScriptPubKey.Addresses = tmp.Vout[i].ScriptPubKey.Addresses + } + } + } + return err +} + +func testGetBlockHash(t *testing.T, h *TestHandler) { + hash, err := h.Client.GetBlockHash(h.TestData.BlockHeight) + if err != nil { + t.Error(err) + return + } + + if hash != h.TestData.BlockHash { + t.Errorf("GetBlockHash() got %q, want %q", hash, h.TestData.BlockHash) + } +} +func testGetBlock(t *testing.T, h *TestHandler) { + blk, err := h.Client.GetBlock(h.TestData.BlockHash, 0) + if err != nil { + t.Error(err) + return + } + + if len(blk.Txs) != len(h.TestData.BlockTxs) { + t.Errorf("GetBlock() number of transactions: got %d, want %d", len(blk.Txs), len(h.TestData.BlockTxs)) + } + + for ti, tx := range blk.Txs { + if tx.Txid != h.TestData.BlockTxs[ti] { + t.Errorf("GetBlock() transaction %d: got %s, want %s", ti, tx.Txid, h.TestData.BlockTxs[ti]) + } + } +} +func testGetTransaction(t *testing.T, h *TestHandler) { + for txid, want := range h.TestData.TxDetails { + got, err := h.Client.GetTransaction(txid) + if err != nil { + t.Error(err) + return + } + // Confirmations is variable field, we just check if is set and reset it + if got.Confirmations <= 0 { + t.Errorf("GetTransaction() got struct with invalid Confirmations field") + continue + } + got.Confirmations = 0 + + if !reflect.DeepEqual(got, want) { + t.Errorf("GetTransaction() got %+v, want %+v", got, want) + } + } +} +func testGetTransactionForMempool(t *testing.T, h *TestHandler) { + for txid, want := range h.TestData.TxDetails { + // reset fields that are not parsed by BlockChainParser + want.Confirmations, want.Blocktime, want.Time = 0, 0, 0 + + got, err := h.Client.GetTransactionForMempool(txid) + if err != nil { + t.Fatal(err) + } + // transactions parsed from JSON may contain additional data + got.Confirmations, got.Blocktime, got.Time = 0, 0, 0 + if !reflect.DeepEqual(got, want) { + t.Errorf("GetTransactionForMempool() got %+v, want %+v", got, want) + } + } +} +func testMempoolSync(t *testing.T, h *TestHandler) { + for i := 0; i < 3; i++ { + txs := getMempool(t, h) + + n, err := h.Client.ResyncMempool(nil) + if err != nil { + t.Fatal(err) + } + if n == 0 { + // no transactions to test + continue + } + + txs = intersect(txs, getMempool(t, h)) + if len(txs) == 0 { + // no transactions to test + continue + } + + txid2addrs := getMempoolAddresses(t, h, txs) + if len(txid2addrs) == 0 { + t.Skip("Skipping test, no addresses in mempool") + } + + for txid, addrs := range txid2addrs { + for _, a := range addrs { + got, err := h.Client.GetMempoolTransactions(a) + if err != nil { + t.Fatal(err) + } + if !containsString(got, txid) { + t.Errorf("ResyncMempool() - for address %s, transaction %s wasn't found in mempool", a, txid) + return + } + } + } + + // done + return + } + t.Skip("Skipping test, all attempts to sync mempool failed due to network state changes") +} +func testEstimateSmartFee(t *testing.T, h *TestHandler) { + for _, blocks := range []int{1, 2, 3, 5, 10} { + fee, err := h.Client.EstimateSmartFee(blocks, true) + if err != nil { + t.Error(err) + } + if fee.Sign() == -1 { + sf := h.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateSmartFee() returned unexpected fee rate: %v", sf) + } + } + } +} +func testEstimateFee(t *testing.T, h *TestHandler) { + for _, blocks := range []int{1, 2, 3, 5, 10} { + fee, err := h.Client.EstimateFee(blocks) + if err != nil { + t.Error(err) + } + if fee.Sign() == -1 { + sf := h.Client.GetChainParser().AmountToDecimalString(&fee) + if sf != "-1" { + t.Errorf("EstimateFee() returned unexpected fee rate: %v", sf) + } + } + } +} +func testGetBestBlockHash(t *testing.T, h *TestHandler) { + for i := 0; i < 3; i++ { + hash, err := h.Client.GetBestBlockHash() + if err != nil { + t.Fatal(err) + } + + height, err := h.Client.GetBestBlockHeight() + if err != nil { + t.Fatal(err) + } + hh, err := h.Client.GetBlockHash(height) + if err != nil { + t.Fatal(err) + } + if hash != hh { + time.Sleep(time.Millisecond * 100) + continue + } + + // we expect no next block + _, err = h.Client.GetBlock("", height+1) + if err != nil { + if err != bchain.ErrBlockNotFound { + t.Error(err) + } + return + } + } + t.Error("GetBestBlockHash() didn't get the best hash") +} +func testGetBestBlockHeight(t *testing.T, h *TestHandler) { + for i := 0; i < 3; i++ { + height, err := h.Client.GetBestBlockHeight() + if err != nil { + t.Fatal(err) + } + + // we expect no next block + _, err = h.Client.GetBlock("", height+1) + if err != nil { + if err != bchain.ErrBlockNotFound { + t.Error(err) + } + return + } + } + t.Error("GetBestBlockHeigh() didn't get the the best heigh") +} +func testGetBlockHeader(t *testing.T, h *TestHandler) { + want := &bchain.BlockHeader{ + Hash: h.TestData.BlockHash, + Height: h.TestData.BlockHeight, + Time: h.TestData.BlockTime, + } + + got, err := h.Client.GetBlockHeader(h.TestData.BlockHash) + if err != nil { + t.Fatal(err) + } + + // Confirmations is variable field, we just check if is set and reset it + if got.Confirmations <= 0 { + t.Fatalf("GetBlockHeader() got struct with invalid Confirmations field") + } + got.Confirmations = 0 + + got.Prev, got.Next = "", "" + + if !reflect.DeepEqual(got, want) { + t.Errorf("GetBlockHeader() got=%+v, want=%+v", got, want) + } +} + +func getMempool(t *testing.T, h *TestHandler) []string { + txs, err := h.Client.GetMempool() + if err != nil { + t.Fatal(err) + } + if len(txs) == 0 { + t.Skip("Skipping test, mempool is empty") + } + + return txs +} + +func getMempoolAddresses(t *testing.T, h *TestHandler, txs []string) map[string][]string { + txid2addrs := map[string][]string{} + for i := 0; i < len(txs); i++ { + tx, err := h.Client.GetTransactionForMempool(txs[i]) + if err != nil { + t.Fatal(err) + } + addrs := []string{} + for _, vin := range tx.Vin { + for _, a := range vin.Addresses { + if isSearchableAddr(a) { + addrs = append(addrs, a) + } + } + } + for _, vout := range tx.Vout { + for _, a := range vout.ScriptPubKey.Addresses { + if isSearchableAddr(a) { + addrs = append(addrs, a) + } + } + } + if len(addrs) > 0 { + txid2addrs[tx.Txid] = addrs + } + } + return txid2addrs +} + +func isSearchableAddr(addr string) bool { + return len(addr) > 3 && addr[:3] != "OP_" +} + +func intersect(a, b []string) []string { + setA := mapset.NewSet() + for _, v := range a { + setA.Add(v) + } + setB := mapset.NewSet() + for _, v := range b { + setB.Add(v) + } + inter := setA.Intersect(setB) + res := make([]string, 0, inter.Cardinality()) + for v := range inter.Iter() { + res = append(res, v.(string)) + } + return res +} + +func containsString(slice []string, s string) bool { + for i := 0; i < len(slice); i++ { + if slice[i] == s { + return true + } + } + return false +} diff --git a/bchain/tests/rpc/testdata/Ethereum_Testnet.json b/bchain/tests/rpc/testdata/Ethereum_Testnet.json deleted file mode 100644 index 82bbf3c7..00000000 --- a/bchain/tests/rpc/testdata/Ethereum_Testnet.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "blockHeight": 2870000, - "blockHash": "0xeccd6b0031015a19cb7d4e10f28590ba65a6a54ad1baa322b50fe5ad16903895", - "blockTxs": [ - "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", - "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - "0xe14e6628e866555091fda44f934dfc6bfbbcd8a81e1281bb682eaf293d248be8", - "0xcffe6d5f4022822ba5f1e64f5b413c7f91c3a48c3739429d72065ec1a8b82e91", - "0x1d261bf23077b7e68ad9300ce9d88e7f4f95075fe98ea76b70534acaf5cf891b", - "0x92f73c26c99ea3b15535829cf686541a1318623baf8c49fe2bf0168bf3c3e272", - "0x392394bb4e4463c9ed59182797b5dbf23aa41c6f6edd7f4b5025d82acf43c357", - "0x3dd838b8d5d9b7155c960f8a138a9c499b87d84b7c9d9a513d8022b1991f959c", - "0x4889b4d1ad3652f6c410a6b6c05a459b86e68363d795f38ec13d5dc6d595d977", - "0xd1e1ff22a80135d904d119252c5d56a8c5b07af9d26de79fd3dda8aeffccf9ae", - "0xd1c3b7835c6032eb9eb07fdce90944f26d470b1c027adf2951b329a783e8e628", - "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0" - ], - "txDetails": { - "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d": { - "hex": "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307830222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - "txid": "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - "blocktime": 1521515026, - "time": 1521515026, - "vin": [ - { - "addresses": ["0xdacc9c61754a0c4616fc5323dc946e89eb272302"] - } - ], - "vout": [ - { - "scriptPubKey": { - "addresses": ["0x682b7903a11098cf770c7aef4aa02a85b3f3601a"] - } - } - ] - }, - "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3": { - "hex": "7b226e6f6e6365223a223078346566222c226761735072696365223a223078626134336237343030222c22676173223a2230783936653838222c22746f223a22307864303534323939633438326164356362333961336661383734373235663965326132306433343963222c2276616c7565223a22307830222c22696e707574223a2230783165626366613630303030303030303030303030303030303030303030303030633034383732383566313736663532306635636434363732306236383662366534366165636430373030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303031303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030313430303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030633461303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303530303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030353831303030303030303030303030303030303030303030303030633365313631343261306432366336616530623163656238643961373236653734653164613664663666656562653137653437306464626434656261663364623938333136386235316337623136363732613131623966306662383864623438323962653237346230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303034363336663665363230303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303437323666373336313030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030222c2268617368223a22307831376565323335666330333539313535623235343139653065346336356439633530306466366537316538323838643665663032306430346363326632636233222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838623461353365353739303739343466633261653539623837373066393331623639326232373062222c227472616e73616374696f6e496e646578223a22307830222c2276223a2230783163222c2272223a22307866633739633836353638323039313030323134616339353662373930653066383935656130343135313438643233613239353632356564393761633936333534222c2273223a223078373232303833616331643764663662626162393939383537616163323561336665646265386130633561326266376364653835393738363266373862313937227d", - "txid": "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", - "blocktime": 1521515026, - "time": 1521515026, - "vin": [ - { - "addresses": ["0x8b4a53e57907944fc2ae59b8770f931b692b270b"] - } - ], - "vout": [ - { - "scriptPubKey": { - "addresses": ["0xd054299c482ad5cb39a3fa874725f9e2a20d349c"] - } - } - ] - } - } -} diff --git a/bchain/tests/rpc/testdata/Bcash.json b/bchain/tests/rpc/testdata/bcash.json similarity index 99% rename from bchain/tests/rpc/testdata/Bcash.json rename to bchain/tests/rpc/testdata/bcash.json index f72df31f..5459546e 100644 --- a/bchain/tests/rpc/testdata/Bcash.json +++ b/bchain/tests/rpc/testdata/bcash.json @@ -1,6 +1,7 @@ { "blockHeight": 538226, "blockHash": "0000000000000000008f48dad3a9efb25275fe36bb31b460e775af57783cc9af", + "blockTime": 1531135480, "blockTxs": [ "77360353ff628d53534201712db7a7689a3f9318cad240fbc093dfb4778fdf84", "82d95972a276b4197806ec0d9e3e56b66523849b2a7597ac509f4dfc8f7db4fd", @@ -70,6 +71,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "fe496933e0a3582ef020bd35c38fe8244a80fa7c63e7607f6f9ccb0806f419e4", @@ -175,6 +177,7 @@ "blocktime":1531135480, "time":1531135480, "locktime": 0, + "version": 1, "vin": [ { "txid": "52f37874fd5a2497c5d84619ab415ca17ebb1d49d559f0d468869f70537354b9", diff --git a/bchain/tests/rpc/testdata/Bcash_Testnet.json b/bchain/tests/rpc/testdata/bcash_testnet.json similarity index 98% rename from bchain/tests/rpc/testdata/Bcash_Testnet.json rename to bchain/tests/rpc/testdata/bcash_testnet.json index 39eba5e7..96b9f476 100644 --- a/bchain/tests/rpc/testdata/Bcash_Testnet.json +++ b/bchain/tests/rpc/testdata/bcash_testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 1241851, "blockHash": "00000000ff54973b45ac277fc9e92408479703d785eae0e96508bc5aa792d621", + "blockTime": 1529571678, "blockTxs": [ "0cbcd28da300a850fd922caf633f2aaca0fd0a2d4ce46c6f07314c644287527a", "32fe2a3a60eb952d6ade53b7030ddb6ad8af5797391857baaad37c5b20f3b5ca" @@ -12,6 +13,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "coinbase": "03fbf212055374617368", @@ -34,6 +36,7 @@ "blocktime":1529571678, "time":1529571678, "locktime": 0, + "version": 1, "vin": [ { "txid": "58ff2450a71b3c228d17d04ec8edcaae452bc97d5ccc4591e2741fd8b031d221", diff --git a/bchain/tests/rpc/testdata/Bitcoin.json b/bchain/tests/rpc/testdata/bitcoin.json similarity index 99% rename from bchain/tests/rpc/testdata/Bitcoin.json rename to bchain/tests/rpc/testdata/bitcoin.json index ba0467cc..ae426860 100644 --- a/bchain/tests/rpc/testdata/Bitcoin.json +++ b/bchain/tests/rpc/testdata/bitcoin.json @@ -1,6 +1,7 @@ { "blockHeight": 529150, "blockHash": "00000000000000000035835503f43c878ebb643f3b40bdfd0dfda760da74e73c", + "blockTime": 1529915213, "blockTxs": [ "8dd1379174e262d12a32d217e87a7caf09fa1b9e48a6fe010cac219f18c6de58", "5fce44793b328ca5f142caadbf29efc78a0059d7a6379dff81fc6447b519a7c3", @@ -120,6 +121,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 0, + "version": 1, "vin": [ { "txid": "83eff5bcc738d52e04528984cd5cf601b69e7df65b2b10ed2475c0072cdde14c", @@ -153,6 +155,7 @@ "blocktime": 1529915213, "time": 1529915213, "locktime": 529149, + "version": 2, "vin": [ { "txid": "413988d546516707f3b20ca9876e026a9472bd814662e42f37f4e57a15713da7", diff --git a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json b/bchain/tests/rpc/testdata/bitcoin_testnet.json similarity index 98% rename from bchain/tests/rpc/testdata/Bitcoin_Testnet.json rename to bchain/tests/rpc/testdata/bitcoin_testnet.json index ece3c941..ed0cde17 100644 --- a/bchain/tests/rpc/testdata/Bitcoin_Testnet.json +++ b/bchain/tests/rpc/testdata/bitcoin_testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 1325168, "blockHash": "000000000000004ed0834f3de922e66d024ec4da9fcc2da17be61369cb6dc041", + "blockTime": 1528788394, "blockTxs": [ "e1179f205aabbf48dc2ce4ebd9ed255571b0578e4de551f6574a50cb81120007", "00a5aa2891d41af9eb1dc30c940f142a609ecab8f370eb0874ba7d32252d1b1b", @@ -43,6 +44,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "4097f5265397047bffe219a1bca65ea5726402c3d9aeecd577fa1c274fc1b8a4", @@ -84,6 +86,7 @@ "blocktime": 1528788394, "time": 1528788394, "locktime": 0, + "version": 1, "vin": [ { "txid": "80437ba4e81bde8b584258f5adad0d08dea60f1e38026344442ad59a4ef797c9", diff --git a/bchain/tests/rpc/testdata/Dash.json b/bchain/tests/rpc/testdata/dash.json similarity index 98% rename from bchain/tests/rpc/testdata/Dash.json rename to bchain/tests/rpc/testdata/dash.json index 150e7d51..d43f66db 100644 --- a/bchain/tests/rpc/testdata/Dash.json +++ b/bchain/tests/rpc/testdata/dash.json @@ -1,6 +1,7 @@ { "blockHeight": 894503, "blockHash": "0000000000000026e2b7c7bb478f0ee846dd976ac6b97c3f3f7c8c65beab88a4", + "blockTime": 1530189699, "blockTxs": [ "6ea3a7fee4c40170d8782719d6fbeafd4cabb830f880d91bcf34811ad79e57bb", "c269940d8e46d94ff61a4adec8aa6dfb13803723a95a5b61c7831d3aea308cbc", @@ -27,6 +28,7 @@ "blocktime": 1530189699, "time": 1530189699, "locktime": 0, + "version": 1, "vin": [ { "txid": "a78824a88c089ea1344fe2a33c454024e6501e76cca2590515b390d85082bd23", diff --git a/bchain/tests/rpc/testdata/Dash_Testnet.json b/bchain/tests/rpc/testdata/dash_testnet.json similarity index 98% rename from bchain/tests/rpc/testdata/Dash_Testnet.json rename to bchain/tests/rpc/testdata/dash_testnet.json index d86f0bee..c8a6299c 100644 --- a/bchain/tests/rpc/testdata/Dash_Testnet.json +++ b/bchain/tests/rpc/testdata/dash_testnet.json @@ -1,7 +1,8 @@ { "blockHeight": 139521, "blockHash": "000000000296ea05e13b5479f6c041de575eec90759f58f57d311a8918b0af17", - "blockTxs": [ + "blockTime": 1528713762, + "blockTxs": [ "cfe2d2c5bd9929349a9d8f3d8f2423f4f0c9b408ed41b3002be5e7437a20aa7d", "3f7ac6c55c0b383feb17715aef3cebf29182039fc42577d7c517b2fc04096942", "3f296b929f6ee4f0ad3d6fce35873d6aeacbdf06aaf2abe384beb6e3f2a4dd03", @@ -31,6 +32,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "a41616c7585c98aeda98d6ff6766b15455e327c9472582b80289dab7597ad309", @@ -64,6 +66,7 @@ "blocktime": 1528713762, "time": 1528713762, "locktime": 139520, + "version": 2, "vin": [ { "txid": "187ae015e41dff766f5a18ae705f59db950a6729a06fa5fd04630c362a9aee27", diff --git a/bchain/tests/rpc/testdata/dogecoin.json b/bchain/tests/rpc/testdata/dogecoin.json new file mode 100644 index 00000000..75b32f85 --- /dev/null +++ b/bchain/tests/rpc/testdata/dogecoin.json @@ -0,0 +1,53 @@ +{ + "blockHeight": 2000002, + "blockHash": "6db0b6fc543cacbc4244f5fab23af56792a87eacc3149957a1922fd59d4d03e0", + "blockTime": 1512601001, + "blockTxs": [ + "cc3c92a9da8e28f18faf17efe74e96f541f03e3a300c55fef0c4dbc9e5b14c01", + "cb3cb43e34385556c9617699687b15f3dbad57cb0e8548114b66c55512c51a52", + "f6be02faa646a1ad764e4eb49fb6f02cbae69e5bc396a8977e59c7d887aa38b2", + "79883da5f57c942feb922e4bb635af631219aae2e70037459b07840945d6466c", + "23b5caddcb3c1222a0b97ff6f67185266b982465ef1a3546c7fc6ad8414ba7d0", + "82cf9f6822362ecfc0d47cc6b7372862ef0fdd14b259c42c8f2def8589f266f0", + "12a4e4daa6e3b842a2395c315f296f284ac2085e2f2fb4f356f94e08117963ba", + "cc136353be3db35e051e5dd34ff29f9cfe47a7b3f6ab7383fd644bc00ab1e348", + "f2db6749bc5f7e727f9e6c9a5553ad44dc59d9994dcadc24a57cf6042d1d07e7" + ], + "txDetails": { + "f6be02faa646a1ad764e4eb49fb6f02cbae69e5bc396a8977e59c7d887aa38b2": { + "hex": "0100000002361742ebea8bdce921359dbe528b23cc8050dc86950fd07011a74258cd0215dde5030000db00483045022100885238254408979e8f44c087c06333376980a73594c4d51c8cdf457934514e34022026287952d554f9f9f24c0a47096cce0db53547099aa700a91fa9046eb5c4960801483045022100a4ce7700bdf654dfe2a4fc5796bd3466be6af3691dc263821d4c71f91f9d9d1e02200b988bea6bf9e7d44e9a4b8cee9e390c72d284e3df1aacf6ab23368560b3e53e0147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52aeffffffff85f2900e642720043fae056075b48ea83cbd3653797c40fb4e7e512938dcda7573020000da004730440220138a8dd272f3081e54b172fb3e6cf43fe5375369d7e93c310ceba852c22ba16f022050de068f932f6d6db9889a8d4e489ffeda2ed1848cd2a71a06d635e6d1799b8e01483045022100af99d412c97062219ef56c3c8ad99023feab94042d310f792392940c61df9bdd02200b4542395624412260c96c46427509c2b801df33e15da08c6c505594c7c142010147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52aeffffffff01c0ebf7b90100000017a9147541523df4d0d0875c024e1906b0d195abaf20958700000000", + "txid": "f6be02faa646a1ad764e4eb49fb6f02cbae69e5bc396a8977e59c7d887aa38b2", + "blocktime": 1512601001, + "time": 1512601001, + "locktime": 0, + "version": 1, + "vin": [ + { + "txid": "dd1502cd5842a71170d00f9586dc5080cc238b52be9d3521e9dc8beaeb421736", + "vout": 997, + "scriptSig": { + "hex": "00483045022100885238254408979e8f44c087c06333376980a73594c4d51c8cdf457934514e34022026287952d554f9f9f24c0a47096cce0db53547099aa700a91fa9046eb5c4960801483045022100a4ce7700bdf654dfe2a4fc5796bd3466be6af3691dc263821d4c71f91f9d9d1e02200b988bea6bf9e7d44e9a4b8cee9e390c72d284e3df1aacf6ab23368560b3e53e0147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52ae" + }, + "sequence": 4294967295 + }, + { + "txid": "75dadc3829517e4efb407c795336bd3ca88eb4756005ae3f042027640e90f285", + "vout": 627, + "scriptSig": { + "hex": "004730440220138a8dd272f3081e54b172fb3e6cf43fe5375369d7e93c310ceba852c22ba16f022050de068f932f6d6db9889a8d4e489ffeda2ed1848cd2a71a06d635e6d1799b8e01483045022100af99d412c97062219ef56c3c8ad99023feab94042d310f792392940c61df9bdd02200b4542395624412260c96c46427509c2b801df33e15da08c6c505594c7c142010147522102a39a18546545ee2944481953c25b3a5f4942cd8ed533ae2daabb4339c68c599b210349ed7f7284feff2b9d0b3493fde313afcc63b41347592c435f5408852c9ea05d52ae" + }, + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 74.15000000, + "n": 0, + "scriptPubKey": { + "hex": "a9147541523df4d0d0875c024e1906b0d195abaf209587" + } + } + ] + } + } +} \ No newline at end of file diff --git a/bchain/tests/rpc/testdata/ethereum_testnet_ropsten.json b/bchain/tests/rpc/testdata/ethereum_testnet_ropsten.json new file mode 100644 index 00000000..a550e3f9 --- /dev/null +++ b/bchain/tests/rpc/testdata/ethereum_testnet_ropsten.json @@ -0,0 +1,40 @@ +{ + "blockHeight": 2870000, + "blockHash": "0xeccd6b0031015a19cb7d4e10f28590ba65a6a54ad1baa322b50fe5ad16903895", + "blockTime": 1521515026, + "blockTxs": [ + "0x17ee235fc0359155b25419e0e4c65d9c500df6e71e8288d6ef020d04cc2f2cb3", + "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", + "0xe14e6628e866555091fda44f934dfc6bfbbcd8a81e1281bb682eaf293d248be8", + "0xcffe6d5f4022822ba5f1e64f5b413c7f91c3a48c3739429d72065ec1a8b82e91", + "0x1d261bf23077b7e68ad9300ce9d88e7f4f95075fe98ea76b70534acaf5cf891b", + "0x92f73c26c99ea3b15535829cf686541a1318623baf8c49fe2bf0168bf3c3e272", + "0x392394bb4e4463c9ed59182797b5dbf23aa41c6f6edd7f4b5025d82acf43c357", + "0x3dd838b8d5d9b7155c960f8a138a9c499b87d84b7c9d9a513d8022b1991f959c", + "0x4889b4d1ad3652f6c410a6b6c05a459b86e68363d795f38ec13d5dc6d595d977", + "0xd1e1ff22a80135d904d119252c5d56a8c5b07af9d26de79fd3dda8aeffccf9ae", + "0xd1c3b7835c6032eb9eb07fdce90944f26d470b1c027adf2951b329a783e8e628", + "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0" + ], + "txDetails": { + "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0": { + "hex": "7b226e6f6e6365223a223078333632306136222c226761735072696365223a223078313261303566323030222c22676173223a22307835323038222c22746f223a22307831623137626331326166623635643563346238316139666632613037366234326131396661616136222c2276616c7565223a223078646530623662336137363430303030222c22696e707574223a223078222c2268617368223a22307837663064313430333239393431663132306235623366633735316533306164656238376232616562626663653561646364303231363630346133346236636330222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307838316237653038663635626466353634383630366338393939386139636338313634333937363437222c227472616e73616374696f6e496e646578223a22307862222c2276223a2230783162222c2272223a22307862666662323864633865373939383833366639356664616139396433626435666635346365663562313839636462383537333537666161326431616231393136222c2273223a22307833616462316365313264396664306538616662373064386639346534636538356137666639346465333966623636333139363638363435663464643138646432227d", + "txid": "0x7f0d140329941f120b5b3fc751e30adeb87b2aebbfce5adcd0216604a34b6cc0", + "blocktime": 1521515026, + "time": 1521515026, + "vin": [ + { + "addresses": ["0x81b7e08f65bdf5648606c89998a9cc8164397647"] + } + ], + "vout": [ + { + "value": 1, + "scriptPubKey": { + "addresses": ["0x1b17bc12afb65d5c4b81a9ff2a076b42a19faaa6"] + } + } + ] + } + } +} diff --git a/bchain/tests/rpc/testdata/litecoin.json b/bchain/tests/rpc/testdata/litecoin.json new file mode 100644 index 00000000..4f5a5f5a --- /dev/null +++ b/bchain/tests/rpc/testdata/litecoin.json @@ -0,0 +1,49 @@ +{ + "blockHeight": 1377592, + "blockHash": "bddb1cfbd474e9516399b373e411bd33c1a71cb01aa8469a27d397ef0a891c7d", + "blockTime": 1519947864, + "blockTxs": [ + "84e9147bf6e171adbda3b3961e467652286d9d9c2933d19326bf84766d047922", + "8d6e628b891dd17bfe3bb5a24a6c7f02ebc2cf499a85515d0325033aa74ab53a", + "5b77ca9735f65d110b086be410658d0239e1fcee13231942a262b35a5b8d6a91", + "19ad3daa2447be4e000d822f79ce252f274b016dacf1418b433d36c0aaf24f18", + "5bffbf0c8ff66d298d94dc323c3644e21932dfc733603d6637ff46cb8d34466c", + "90d587e35b23905f0125111f41d69bb6c7eed44f0944caad2903aae1f174ac49" + ], + "txDetails": { + "19ad3daa2447be4e000d822f79ce252f274b016dacf1418b433d36c0aaf24f18": { + "hex": "010000000001011f9216c16c78386540d7ae7d32657c388ef5f204596f84ea0851dcb78c479a87010000001716001432d094c8e2efd308d2c69affd6712ebbf7a5a286ffffffff0289544a110000000017a91457ca840d6c811dd6808722babe3f88d2fdb2ca14875bb1c9180000000017a91482696a9b4188eda8f93b120315cad4260cbb90db8702483045022100aa256153317133fa719180935017671e33ca77df6f5426554f5b0855f07a392b02202684e8c30623e1c4e753ea23a95bd571e1fda7d15dc0b1e2d54ff5bc50329256012103399a1d98f0733ef400ff8d4f43fe4543065f7f387c863361c77a8826321ca6fb00000000", + "txid": "19ad3daa2447be4e000d822f79ce252f274b016dacf1418b433d36c0aaf24f18", + "blocktime": 1519947864, + "time": 1519947864, + "locktime": 0, + "version": 1, + "vin": [ + { + "txid": "879a478cb7dc5108ea846f5904f2f58e387c65327daed7406538786cc116921f", + "vout": 1, + "scriptSig": { + "hex": "16001432d094c8e2efd308d2c69affd6712ebbf7a5a286" + }, + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 2.90083977, + "n": 0, + "scriptPubKey": { + "hex": "a91457ca840d6c811dd6808722babe3f88d2fdb2ca1487" + } + }, + { + "value": 4.15871323, + "n": 1, + "scriptPubKey": { + "hex": "a91482696a9b4188eda8f93b120315cad4260cbb90db87" + } + } + ] + } + } +} \ No newline at end of file diff --git a/bchain/tests/rpc/testdata/monacoin.json b/bchain/tests/rpc/testdata/monacoin.json new file mode 100644 index 00000000..2e78ebe6 --- /dev/null +++ b/bchain/tests/rpc/testdata/monacoin.json @@ -0,0 +1,55 @@ +{ + "blockHeight": 1205643, + "blockHash": "60c80ce4f0a90f7b217bef8fab32ce2e499a4dc2067f4a6062048e3b940d76c6", + "blockTime": 1514835214, + "blockTxs": [ + "38b4a919ab0f8b4570e1f26717740846b7994ba22f94c51d96eb32bc45c5886e", + "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298", + "c2bdcc759c1d4e6bb3eee859cc7400bc376479e30bc8328f31864aad92d52ade", + "8a26deeda15f1ddc3234acafb0da9829bf334ea108e359f11d8f281ae42512f8" + ], + "txDetails": { + "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298": { + "hex": "0200000002d9d3592e550618b41347bd992d33b9eaebe7e26eb21df49bda5372ce1711f8e5010000006b483045022100b4fb223759753d82fd5ce833e84f62904c086b51fd2be11d7e3e9f183e2442850220079f8b3d6555610ff98da3216fa117bcb49f98e610a38471d69566ce7bc5770801210362c5e60c40f0b588216310dbb2f4a46e86038ba597f33d023ad64e66ff2e972bfeffffff4f4e9b373e6cd134d2f5f45bf3645a00c2689472ad66aa37ae022eef3dd6b16a010000006a47304402203dd8510301d68bc21169f953d2996d602f8411a309c494392b6f2bf85e5d58f102206fa2db245d15021f3b556d10747cfd8eba100fd19dfa1a0a9c5cebc58d7c22f6012103c65394a4c6705e3a13f6305e7997fb68feb192acce312a8ba144994b9a33507bfeffffff02bbe52c00000000001976a914eb0f5829a547f30b3d488ab38b425c1d93afb4b888ac00f90295000000001976a914883f0723e1c93db538d02c7a8d318322e655b59888ac89651200", + "txid": "7f793aff2f74ca9cec6df08a7c0fcaa1ea301468459b46edbed693a871832298", + "blocktime": 1514835214, + "time": 1514835214, + "locktime": 1205641, + "version": 2, + "vin": [ + { + "txid": "e5f81117ce7253da9bf41db26ee2e7ebeab9332d99bd4713b41806552e59d3d9", + "vout": 1, + "scriptSig": { + "hex": "483045022100b4fb223759753d82fd5ce833e84f62904c086b51fd2be11d7e3e9f183e2442850220079f8b3d6555610ff98da3216fa117bcb49f98e610a38471d69566ce7bc5770801210362c5e60c40f0b588216310dbb2f4a46e86038ba597f33d023ad64e66ff2e972b" + }, + "sequence": 4294967294 + }, + { + "txid": "6ab1d63def2e02ae37aa66ad729468c2005a64f35bf4f5d234d16c3e379b4e4f", + "vout": 1, + "scriptSig": { + "hex": "47304402203dd8510301d68bc21169f953d2996d602f8411a309c494392b6f2bf85e5d58f102206fa2db245d15021f3b556d10747cfd8eba100fd19dfa1a0a9c5cebc58d7c22f6012103c65394a4c6705e3a13f6305e7997fb68feb192acce312a8ba144994b9a33507b" + }, + "sequence": 4294967294 + } + ], + "vout": [ + { + "value": 0.02942395, + "n": 0, + "scriptPubKey": { + "hex": "76a914eb0f5829a547f30b3d488ab38b425c1d93afb4b888ac" + } + }, + { + "value": 25.00000000, + "n": 1, + "scriptPubKey": { + "hex": "76a914883f0723e1c93db538d02c7a8d318322e655b59888ac" + } + } + ] + } + } +} \ No newline at end of file diff --git a/bchain/tests/rpc/testdata/Namecoin.json b/bchain/tests/rpc/testdata/namecoin.json similarity index 98% rename from bchain/tests/rpc/testdata/Namecoin.json rename to bchain/tests/rpc/testdata/namecoin.json index 4e1193cd..489f0c53 100644 --- a/bchain/tests/rpc/testdata/Namecoin.json +++ b/bchain/tests/rpc/testdata/namecoin.json @@ -1,6 +1,7 @@ { "blockHeight": 404680, "blockHash": "920fe53b840111f7e593d93ba58dc54e043e10f8fa4a678e86a98f5cb5b29614", + "blockTime": 1530003649, "blockTxs": [ "80b8477d10df9ece7d8dde2d30817e2855af1fb66b7a9ac860e592118ae33f5f", "afcd8e3638b11b1ce52055474dcce78c4129e95ebee3305574acda48deea8a65", @@ -39,6 +40,7 @@ "blocktime": 1530003649, "time": 1530003649, "locktime": 404678, + "version": 1, "vin": [ { "txid": "223a4ff7a613173ffbc67f9efffe1b0c3fd5ba2471935a44cc81814038272901", diff --git a/bchain/tests/rpc/testdata/Vertcoin.json b/bchain/tests/rpc/testdata/vertcoin.json similarity index 98% rename from bchain/tests/rpc/testdata/Vertcoin.json rename to bchain/tests/rpc/testdata/vertcoin.json index 021cdc2a..fb18cacb 100644 --- a/bchain/tests/rpc/testdata/Vertcoin.json +++ b/bchain/tests/rpc/testdata/vertcoin.json @@ -1,6 +1,7 @@ { "blockHeight": 952235, "blockHash": "b2787dd022e3aa65b63dbf08af2c9bb4d4a362d95e3328c02743a5c8d75acb36", + "blockTime": 1529932850, "blockTxs": [ "366eca05fa8579465d8822ad6462762120b26239201a34981e5f9d9efac3cc31", "e74c247a5a77d4edd96a5dbb2930c74d9ab550affde991731f78f3e3a2f4b559", @@ -14,6 +15,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952232, + "version": 2, "vin": [ { "txid": "8ec70404bdd16559c589736863ee9d9f69431ba4e6b2ab3b1641b3f9f3821624", @@ -47,6 +49,7 @@ "blocktime": 1529932850, "time": 1529932850, "locktime": 952233, + "version": 1, "vin": [ { "txid": "5765db7dd62cddada154b2c95505e298818a308cc1ce1e28d6e8565301cb8d74", diff --git a/bchain/tests/rpc/testdata/Zcash.json b/bchain/tests/rpc/testdata/zcash.json similarity index 99% rename from bchain/tests/rpc/testdata/Zcash.json rename to bchain/tests/rpc/testdata/zcash.json index 6b620537..f8b83f80 100644 --- a/bchain/tests/rpc/testdata/Zcash.json +++ b/bchain/tests/rpc/testdata/zcash.json @@ -1,6 +1,7 @@ { "blockHeight": 349410, "blockHash": "000000000101b45343aeda139d06d7f188393a5cf4adabfa191cb1828d67f6bd", + "blockTime": 1530264033, "blockTxs": [ "6ebf0114454ba776f878cf726be3cb65964b21fad8dfca5d2ee69bd4a32cfa97", "3de894528d5f8dbe14f726244be70a88f9b738020215c91b2893fa5a46f48c14", @@ -22,6 +23,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 349399, + "version": 3, "vin": [ { "txid": "9c9faac29b0fa1b0e683727f2973bfcb87e4baadd07e9c997b431a31713cb30c", @@ -57,6 +59,7 @@ "blocktime": 1530264033, "time": 1530264033, "locktime": 0, + "version": 3, "vin": [ { "txid": "4440c8e9d5b57da7ca0fb3f62ec13f392267aeb797c123bb01e850adf8573dd0", diff --git a/bchain/tests/rpc/testdata/Zcash_Testnet.json b/bchain/tests/rpc/testdata/zcash_testnet.json similarity index 98% rename from bchain/tests/rpc/testdata/Zcash_Testnet.json rename to bchain/tests/rpc/testdata/zcash_testnet.json index aca84131..8ed594c1 100644 --- a/bchain/tests/rpc/testdata/Zcash_Testnet.json +++ b/bchain/tests/rpc/testdata/zcash_testnet.json @@ -1,6 +1,7 @@ { "blockHeight": 251102, "blockHash": "001335906f981bbf0633e124e2fa8afef3d882e34a0306a4e0c55162e57e673d", + "blockTime": 1528781777, "blockTxs": [ "f02aa1c4c86e1d0cef6ccbbc48b2b7b38355bc3612d8f77dd58d04be1ec6ba19", "a9f7cc34d7e272d2d9fb68cfa1c1941e338f377e6e426ae2fea1c12616d89c63", @@ -13,6 +14,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251028, + "version": 3, "vin": [ { "txid": "19a1d013b898239e9a2943faa07f8716b9be168bc8e001daf3625f535fde1a60", @@ -48,6 +50,7 @@ "blocktime": 1528781777, "time": 1528781777, "locktime": 251090, + "version": 3, "vin": [ { "txid": "9acab5f13cf94074e75f5686b59fccd938f54b5f20ddddfcb6077c679a13c0ea", diff --git a/bchain/tx.pb.go b/bchain/tx.pb.go index dc5a0952..2115993a 100644 --- a/bchain/tx.pb.go +++ b/bchain/tx.pb.go @@ -148,7 +148,7 @@ func (m *ProtoTransaction_VinType) GetAddresses() []string { } type ProtoTransaction_VoutType struct { - Value float64 `protobuf:"fixed64,1,opt,name=Value" json:"Value,omitempty"` + ValueSat []byte `protobuf:"bytes,1,opt,name=ValueSat,proto3" json:"ValueSat,omitempty"` N uint32 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` ScriptPubKeyHex []byte `protobuf:"bytes,3,opt,name=ScriptPubKeyHex,proto3" json:"ScriptPubKeyHex,omitempty"` Addresses []string `protobuf:"bytes,4,rep,name=Addresses" json:"Addresses,omitempty"` @@ -159,11 +159,11 @@ func (m *ProtoTransaction_VoutType) String() string { return proto.Co func (*ProtoTransaction_VoutType) ProtoMessage() {} func (*ProtoTransaction_VoutType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 1} } -func (m *ProtoTransaction_VoutType) GetValue() float64 { +func (m *ProtoTransaction_VoutType) GetValueSat() []byte { if m != nil { - return m.Value + return m.ValueSat } - return 0 + return nil } func (m *ProtoTransaction_VoutType) GetN() uint32 { @@ -196,26 +196,26 @@ func init() { func init() { proto.RegisterFile("tx.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 330 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xc1, 0x4e, 0xc2, 0x40, - 0x10, 0x86, 0xb3, 0xb4, 0x14, 0x18, 0x21, 0x92, 0x89, 0x31, 0x0d, 0xf1, 0x50, 0x39, 0xf5, 0xd4, - 0x03, 0xc6, 0x07, 0x50, 0x2f, 0x24, 0x1a, 0x42, 0x16, 0xd2, 0x7b, 0x5b, 0x36, 0xb0, 0x11, 0x77, - 0xb1, 0xdd, 0x1a, 0x78, 0x16, 0x1f, 0xc1, 0x97, 0x34, 0x3b, 0x2d, 0x45, 0x48, 0xbc, 0xed, 0xff, - 0xef, 0x4c, 0xff, 0x6f, 0xff, 0x14, 0xba, 0x66, 0x1f, 0xed, 0x72, 0x6d, 0x34, 0x7a, 0x69, 0xb6, - 0x49, 0xa4, 0x1a, 0x7f, 0xbb, 0x30, 0x9c, 0x5b, 0x67, 0x99, 0x27, 0xaa, 0x48, 0x32, 0x23, 0xb5, - 0x42, 0x04, 0x77, 0xb9, 0x97, 0x2b, 0x9f, 0x05, 0x2c, 0xec, 0x73, 0x3a, 0xe3, 0x10, 0x9c, 0xa9, - 0xd8, 0xfb, 0x2d, 0xb2, 0xec, 0x11, 0xef, 0xa0, 0xf7, 0xbc, 0xd5, 0xd9, 0xbb, 0x91, 0x1f, 0xc2, - 0x77, 0x02, 0x16, 0xba, 0xfc, 0x64, 0xe0, 0x08, 0xba, 0x6f, 0xc7, 0x4b, 0x37, 0x60, 0xe1, 0x80, - 0x37, 0x1a, 0x6f, 0xc1, 0x9b, 0x0a, 0xb9, 0xde, 0x18, 0xbf, 0x4d, 0x37, 0xb5, 0xc2, 0x09, 0x38, - 0xb1, 0x54, 0xbe, 0x17, 0x38, 0xe1, 0xd5, 0x24, 0x88, 0x2a, 0xc4, 0xe8, 0x12, 0x2f, 0x8a, 0xa5, - 0x5a, 0x1e, 0x76, 0x82, 0xdb, 0x61, 0x7c, 0x04, 0x37, 0xd6, 0xa5, 0xf1, 0x3b, 0xb4, 0x74, 0xff, - 0xff, 0x92, 0x2e, 0x0d, 0x6d, 0xd1, 0xf8, 0xe8, 0x87, 0x41, 0xa7, 0xfe, 0x8e, 0x45, 0x7d, 0xd1, - 0x52, 0xa5, 0x49, 0x21, 0xe8, 0xc9, 0x3d, 0xde, 0xe8, 0xa6, 0x8a, 0xd6, 0x9f, 0x2a, 0xb0, 0x8e, - 0x74, 0x08, 0x9e, 0xce, 0x38, 0x86, 0xfe, 0x22, 0xcb, 0xe5, 0xce, 0x2c, 0xe4, 0xda, 0xf6, 0xe4, - 0xd2, 0xfc, 0x99, 0x67, 0x73, 0x16, 0xe2, 0xb3, 0x14, 0x2a, 0x13, 0xf5, 0xc3, 0x1b, 0x6d, 0xcb, - 0x7c, 0x5a, 0xad, 0x72, 0x51, 0x14, 0xa2, 0xa0, 0x02, 0x7a, 0xfc, 0x64, 0x8c, 0xbe, 0xa0, 0x7b, - 0xe4, 0xc7, 0x1b, 0x68, 0xc7, 0xc9, 0xb6, 0xac, 0x50, 0x19, 0xaf, 0x04, 0xf6, 0x81, 0xcd, 0x08, - 0x72, 0xc0, 0xd9, 0x0c, 0x43, 0xb8, 0xae, 0x92, 0xe7, 0x65, 0xfa, 0x2a, 0x0e, 0x16, 0xc8, 0x21, - 0xa0, 0x4b, 0xfb, 0x3c, 0xd7, 0xbd, 0xc8, 0x4d, 0x3d, 0xfa, 0x59, 0x1e, 0x7e, 0x03, 0x00, 0x00, - 0xff, 0xff, 0x64, 0x1d, 0x38, 0xac, 0x38, 0x02, 0x00, 0x00, + // 331 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xc1, 0x6e, 0xea, 0x30, + 0x10, 0x94, 0x89, 0x5f, 0x80, 0x7d, 0xa0, 0x87, 0xf6, 0xf0, 0x14, 0xa1, 0x1e, 0x52, 0x4e, 0x39, + 0xe5, 0x40, 0xd5, 0x0f, 0x68, 0x7b, 0x41, 0x6a, 0x85, 0x90, 0x83, 0x72, 0x4f, 0x82, 0x05, 0x56, + 0xa9, 0x4d, 0x13, 0x47, 0x02, 0xa9, 0x3f, 0xd3, 0x73, 0x7f, 0xb2, 0xf2, 0x12, 0x42, 0x41, 0xea, + 0x6d, 0x67, 0xbc, 0xe3, 0x19, 0x4f, 0x02, 0x3d, 0xbb, 0x8f, 0x77, 0xa5, 0xb1, 0x06, 0xfd, 0xbc, + 0xd8, 0x64, 0x4a, 0x4f, 0x3e, 0x39, 0x8c, 0x16, 0x8e, 0x59, 0x96, 0x99, 0xae, 0xb2, 0xc2, 0x2a, + 0xa3, 0x11, 0x81, 0x2f, 0xf7, 0x6a, 0x15, 0xb0, 0x90, 0x45, 0x03, 0x41, 0x33, 0x8e, 0xc0, 0x9b, + 0xc9, 0x7d, 0xd0, 0x21, 0xca, 0x8d, 0x78, 0x03, 0xfd, 0xc7, 0xad, 0x29, 0x5e, 0xad, 0x7a, 0x93, + 0x81, 0x17, 0xb2, 0x88, 0x8b, 0x33, 0x81, 0x63, 0xe8, 0xbd, 0x9c, 0x0e, 0x79, 0xc8, 0xa2, 0xa1, + 0x68, 0x31, 0xfe, 0x07, 0x7f, 0x26, 0xd5, 0x7a, 0x63, 0x83, 0x3f, 0x74, 0xd2, 0x20, 0x9c, 0x82, + 0x97, 0x2a, 0x1d, 0xf8, 0xa1, 0x17, 0xfd, 0x9d, 0x86, 0xf1, 0x31, 0x62, 0x7c, 0x1d, 0x2f, 0x4e, + 0x95, 0x5e, 0x1e, 0x76, 0x52, 0xb8, 0x65, 0xbc, 0x07, 0x9e, 0x9a, 0xda, 0x06, 0x5d, 0x12, 0xdd, + 0xfe, 0x2e, 0x32, 0xb5, 0x25, 0x15, 0xad, 0x8f, 0xbf, 0x18, 0x74, 0x9b, 0x7b, 0x5c, 0xd4, 0x27, + 0xa3, 0x74, 0x9e, 0x55, 0x92, 0x9e, 0xdc, 0x17, 0x2d, 0x6e, 0xab, 0xe8, 0xfc, 0xa8, 0x02, 0x1b, + 0x4b, 0x8f, 0xc2, 0xd3, 0x8c, 0x13, 0x18, 0x24, 0x45, 0xa9, 0x76, 0x36, 0x51, 0x6b, 0xd7, 0x13, + 0xa7, 0xfd, 0x0b, 0xce, 0xf9, 0x24, 0xf2, 0xbd, 0x96, 0xba, 0x90, 0xcd, 0xc3, 0x5b, 0xec, 0xca, + 0x7c, 0x58, 0xad, 0x4a, 0x59, 0x55, 0xb2, 0xa2, 0x02, 0xfa, 0xe2, 0x4c, 0x8c, 0x3f, 0xa0, 0x77, + 0xca, 0xef, 0x6e, 0x49, 0xb3, 0x6d, 0x2d, 0x93, 0xcc, 0x36, 0x1f, 0xa8, 0xc5, 0x38, 0x00, 0x36, + 0xa7, 0xa8, 0x43, 0xc1, 0xe6, 0x18, 0xc1, 0xbf, 0xa3, 0xff, 0xa2, 0xce, 0x9f, 0xe5, 0xc1, 0xc5, + 0xf2, 0x48, 0x70, 0x4d, 0x5f, 0xba, 0xf3, 0x2b, 0xf7, 0xdc, 0xa7, 0x5f, 0xe6, 0xee, 0x3b, 0x00, + 0x00, 0xff, 0xff, 0x95, 0x9e, 0x00, 0x7a, 0x3e, 0x02, 0x00, 0x00, } diff --git a/bchain/tx.proto b/bchain/tx.proto index 731b8bf5..61e06aed 100644 --- a/bchain/tx.proto +++ b/bchain/tx.proto @@ -11,7 +11,7 @@ syntax = "proto3"; repeated string Addresses = 6; } message VoutType { - double Value = 1; + bytes ValueSat = 1; uint32 N = 2; bytes ScriptPubKeyHex = 3; repeated string Addresses = 4; diff --git a/bchain/types.go b/bchain/types.go index c2d350bd..4728a4bf 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -2,9 +2,11 @@ package bchain import ( "context" + "encoding/hex" "encoding/json" "errors" "fmt" + "math/big" ) // errors with specific meaning returned by blockchain rpc @@ -42,17 +44,11 @@ type ScriptPubKey struct { Addresses []string `json:"addresses"` } -type Address interface { - String() string - AreEqual(addr string) bool - InSlice(addrs []string) bool -} - type Vout struct { - Value float64 `json:"value"` + ValueSat big.Int + JsonValue json.Number `json:"value"` N uint32 `json:"n"` ScriptPubKey ScriptPubKey `json:"scriptPubKey"` - Address Address } // Tx is blockchain transaction @@ -75,32 +71,57 @@ type Block struct { Txs []Tx `json:"tx"` } -type ThinBlock struct { - BlockHeader - Txids []string `json:"tx"` -} - +// BlockHeader contains limited data (as needed for indexing) from backend block header type BlockHeader struct { Hash string `json:"hash"` Prev string `json:"previousblockhash"` Next string `json:"nextblockhash"` Height uint32 `json:"height"` Confirmations int `json:"confirmations"` + Size int `json:"size"` + Time int64 `json:"time,omitempty"` +} + +// BlockInfo contains extended block header data and a list of block txids +type BlockInfo struct { + BlockHeader + Version json.Number `json:"version"` + MerkleRoot string `json:"merkleroot"` + Nonce json.Number `json:"nonce"` + Bits string `json:"bits"` + Difficulty json.Number `json:"difficulty"` + Txids []string `json:"tx,omitempty"` } type MempoolEntry struct { - Size uint32 `json:"size"` - Fee float64 `json:"fee"` - ModifiedFee float64 `json:"modifiedfee"` - Time float64 `json:"time"` - Height uint32 `json:"height"` - DescendantCount uint32 `json:"descendantcount"` - DescendantSize uint32 `json:"descendantsize"` - DescendantFees uint32 `json:"descendantfees"` - AncestorCount uint32 `json:"ancestorcount"` - AncestorSize uint32 `json:"ancestorsize"` - AncestorFees uint32 `json:"ancestorfees"` - Depends []string `json:"depends"` + Size uint32 `json:"size"` + FeeSat big.Int + Fee json.Number `json:"fee"` + ModifiedFeeSat big.Int + ModifiedFee json.Number `json:"modifiedfee"` + Time uint64 `json:"time"` + Height uint32 `json:"height"` + DescendantCount uint32 `json:"descendantcount"` + DescendantSize uint32 `json:"descendantsize"` + DescendantFees uint32 `json:"descendantfees"` + AncestorCount uint32 `json:"ancestorcount"` + AncestorSize uint32 `json:"ancestorsize"` + AncestorFees uint32 `json:"ancestorfees"` + Depends []string `json:"depends"` +} + +type ChainInfo struct { + Chain string `json:"chain"` + Blocks int `json:"blocks"` + Headers int `json:"headers"` + Bestblockhash string `json:"bestblockhash"` + Difficulty string `json:"difficulty"` + SizeOnDisk int64 `json:"size_on_disk"` + Version string `json:"version"` + Subversion string `json:"subversion"` + ProtocolVersion string `json:"protocolversion"` + Timeoffset float64 `json:"timeoffset"` + Warnings string `json:"warnings"` } type RPCError struct { @@ -112,6 +133,13 @@ func (e *RPCError) Error() string { return fmt.Sprintf("%d: %s", e.Code, e.Message) } +// AddressDescriptor is an opaque type obtained by parser.GetAddrDesc* methods +type AddressDescriptor []byte + +func (ad AddressDescriptor) String() string { + return "ad:" + hex.EncodeToString(ad) +} + // OnNewBlockFunc is used to send notification about a new block type OnNewBlockFunc func(hash string, height uint32) @@ -128,22 +156,24 @@ type BlockChain interface { GetNetworkName() string GetSubversion() string GetCoinName() string + GetChainInfo() (*ChainInfo, error) // requests - GetBlockChainInfo() (string, error) GetBestBlockHash() (string, error) GetBestBlockHeight() (uint32, error) GetBlockHash(height uint32) (string, error) GetBlockHeader(hash string) (*BlockHeader, error) GetBlock(hash string, height uint32) (*Block, error) + GetBlockInfo(hash string) (*BlockInfo, error) GetMempool() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) - EstimateSmartFee(blocks int, conservative bool) (float64, error) - EstimateFee(blocks int) (float64, error) + EstimateSmartFee(blocks int, conservative bool) (big.Int, error) + EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) // mempool ResyncMempool(onNewTxAddr OnNewTxAddrFunc) (int, error) GetMempoolTransactions(address string) ([]string, error) + GetMempoolTransactionsForAddrDesc(addrDesc AddressDescriptor) ([]string, error) GetMempoolEntry(txid string) (*MempoolEntry, error) // parser GetChainParser() BlockChainParser @@ -151,7 +181,7 @@ type BlockChain interface { // BlockChainParser defines common interface to parsing and conversions of block chain data type BlockChainParser interface { - // self description + // chain configuration description // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db IsUTXOChain() bool @@ -159,12 +189,16 @@ type BlockChainParser interface { // and used in case of fork // if 0 the blockaddresses column is not used at all (usually non UTXO chains) KeepBlockAddresses() int - // address id conversions - GetAddrIDFromVout(output *Vout) ([]byte, error) - GetAddrIDFromAddress(address string) ([]byte, error) - // address to output script conversions - AddressToOutputScript(address string) ([]byte, error) - OutputScriptToAddresses(script []byte) ([]string, error) + // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place + AmountToDecimalString(a *big.Int) string + // AmountToBigInt converts amount in json.Number (string) to big.Int + // it uses string operations to avoid problems with rounding + AmountToBigInt(n json.Number) (big.Int, error) + // address descriptor conversions + GetAddrDescFromVout(output *Vout) (AddressDescriptor, error) + GetAddrDescFromAddress(address string) (AddressDescriptor, error) + GetAddressesFromAddrDesc(addrDesc AddressDescriptor) ([]string, bool, error) + GetScriptFromAddrDesc(addrDesc AddressDescriptor) ([]byte, error) // transactions PackedTxidLen() int PackTxid(txid string) ([]byte, error) diff --git a/blockbook.go b/blockbook.go index 1de45946..41fc8a73 100644 --- a/blockbook.go +++ b/blockbook.go @@ -1,10 +1,18 @@ package main import ( + "blockbook/api" + "blockbook/bchain" + "blockbook/bchain/coins" + "blockbook/common" + "blockbook/db" + "blockbook/server" "context" "flag" "log" "math/rand" + "net/http" + _ "net/http/pprof" "os" "os/signal" "strings" @@ -12,18 +20,9 @@ import ( "syscall" "time" - "github.com/juju/errors" - - "blockbook/bchain" - "blockbook/bchain/coins" - "blockbook/common" - "blockbook/db" - "blockbook/server" - + // "github.com/erikdubbelboer/gspt" "github.com/golang/glog" - - "net/http" - _ "net/http/pprof" + "github.com/juju/errors" ) // debounce too close requests for resync @@ -38,7 +37,9 @@ const storeInternalStatePeriodMs = 59699 var ( blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") - dbPath = flag.String("datadir", "./data", "path to database directory") + dbPath = flag.String("datadir", "./data", "path to database directory") + dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") + dbMaxOpenFiles = flag.Int("dbmaxopenfiles", 1<<14, "max open files by rocksdb") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") @@ -50,15 +51,17 @@ var ( repair = flag.Bool("repair", false, "repair the database") prof = flag.String("prof", "", "http server binding [address]:port of the interface to profiling data /debug/pprof/ (default no profiling)") - syncChunk = flag.Int("chunk", 100, "block chunk size for processing") - syncWorkers = flag.Int("workers", 8, "number of workers to process blocks") + syncChunk = flag.Int("chunk", 100, "block chunk size for processing in bulk mode") + syncWorkers = flag.Int("workers", 8, "number of workers to process blocks in bulk mode") dryRun = flag.Bool("dryrun", false, "do not index blocks, only download") + debugMode = flag.Bool("debug", false, "debug mode, return more verbose errors, reload templates on each request") + internalBinding = flag.String("internal", "", "internal http server binding [address]:port, (default no internal server)") - publicBinding = flag.String("public", "", "public http server binding [address]:port[/path], (default no public server)") + publicBinding = flag.String("public", "", "public http server binding [address]:port[/path] (default no public server)") - certFiles = flag.String("certfile", "", "to enable SSL specify path to certificate files without extension, expecting .crt and .key, (default no SSL)") + certFiles = flag.String("certfile", "", "to enable SSL specify path to certificate files without extension, expecting .crt and .key (default no SSL)") explorerURL = flag.String("explorer", "", "address of blockchain explorer") @@ -83,6 +86,7 @@ var ( chain bchain.BlockChain index *db.RocksDB txCache *db.TxCache + metrics *common.Metrics syncWorker *db.SyncWorker internalState *common.InternalState callbacksOnNewBlock []bchain.OnNewBlockFunc @@ -129,7 +133,7 @@ func main() { chanOsSignal = make(chan os.Signal, 1) signal.Notify(chanOsSignal, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) - glog.Infof("Blockbook: %+v", common.GetVersionInfo()) + glog.Infof("Blockbook: %+v, debug mode %v", common.GetVersionInfo(), *debugMode) if *prof != "" { go func() { @@ -153,7 +157,9 @@ func main() { glog.Fatal("config: ", err) } - metrics, err := common.GetMetrics(coin) + // gspt.SetProcTitle("blockbook-" + normalizeName(coin)) + + metrics, err = common.GetMetrics(coin) if err != nil { glog.Fatal("metrics: ", err) } @@ -162,7 +168,7 @@ func main() { glog.Fatal("rpc: ", err) } - index, err = db.NewRocksDB(*dbPath, chain.GetChainParser(), metrics) + index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics) if err != nil { glog.Fatal("rocksDB: ", err) } @@ -175,7 +181,11 @@ func main() { } index.SetInternalState(internalState) if internalState.DbState != common.DbStateClosed { - glog.Warning("internalState: database in not closed state ", internalState.DbState, ", possibly previous ungraceful shutdown") + if internalState.DbState == common.DbStateInconsistent { + glog.Error("internalState: database is in inconsistent state and cannot be used") + return + } + glog.Warning("internalState: database was left in open state, possibly previous ungraceful shutdown") } if *computeColumnStats { @@ -232,6 +242,11 @@ func main() { return } + // report BlockbookAppInfo metric, only log possible error + if err = blockbookAppInfoMetric(index, chain, txCache, internalState, metrics); err != nil { + glog.Error("blockbookAppInfoMetric ", err) + } + var internalServer *server.InternalServer if *internalBinding != "" { internalServer, err = server.NewInternalServer(*internalBinding, *certFiles, index, chain, txCache, internalState) @@ -252,20 +267,10 @@ func main() { }() } - if *synchronize { - if err := syncWorker.ResyncIndex(nil); err != nil { - glog.Error("resyncIndex ", err) - return - } - if _, err = chain.ResyncMempool(nil); err != nil { - glog.Error("resyncMempool ", err) - return - } - } - var publicServer *server.PublicServer if *publicBinding != "" { - publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, txCache, *explorerURL, metrics, internalState) + // start public server in limited functionality, extend it after sync is finished by calling ConnectFullPublicInterface + publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, txCache, *explorerURL, metrics, internalState, *debugMode) if err != nil { glog.Error("socketio: ", err) return @@ -286,10 +291,27 @@ func main() { } if *synchronize { - // start the synchronization loops after the server interfaces are started + internalState.SyncMode = true + internalState.InitialSync = true + if err := syncWorker.ResyncIndex(nil); err != nil { + glog.Error("resyncIndex ", err) + return + } + var mempoolCount int + if mempoolCount, err = chain.ResyncMempool(nil); err != nil { + glog.Error("resyncMempool ", err) + return + } + internalState.FinishedMempoolSync(mempoolCount) go syncIndexLoop() go syncMempoolLoop() - go storeInternalStateLoop() + internalState.InitialSync = false + } + go storeInternalStateLoop() + + if *publicBinding != "" { + // start full public interface + publicServer.ConnectFullPublicInterface() } if *blockFrom >= 0 { @@ -327,6 +349,25 @@ func main() { } } +func blockbookAppInfoMetric(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState, metrics *common.Metrics) error { + api, err := api.NewWorker(db, chain, txCache, is) + if err != nil { + return err + } + si, err := api.GetSystemInfo(false) + if err != nil { + return err + } + metrics.BlockbookAppInfo.Reset() + metrics.BlockbookAppInfo.With(common.Labels{ + "blockbook_version": si.Blockbook.Version, + "blockbook_commit": si.Blockbook.GitCommit, + "backend_version": si.Backend.Version, + "backend_subversion": si.Backend.Subversion, + "backend_protocol_version": si.Backend.ProtocolVersion}).Set(float64(0)) + return nil +} + func newInternalState(coin string, coinShortcut string, d *db.RocksDB) (*common.InternalState, error) { is, err := d.LoadInternalState(coin) if err != nil { @@ -424,6 +465,8 @@ func storeInternalStateLoop() { lastCompute := time.Now() // randomize the duration between ComputeInternalStateColumnStats to avoid peaks after reboot of machine with multiple blockbooks computePeriod := 9*time.Hour + time.Duration(rand.Float64()*float64((2*time.Hour).Nanoseconds())) + lastAppInfo := time.Now() + logAppInfoPeriod := 15 * time.Minute glog.Info("storeInternalStateLoop starting with db stats recompute period ", computePeriod) tickAndDebounce(storeInternalStatePeriodMs*time.Millisecond, (storeInternalStatePeriodMs-1)*time.Millisecond, chanStoreInternalState, func() { if !computeRunning && lastCompute.Add(computePeriod).Before(time.Now()) { @@ -440,6 +483,13 @@ func storeInternalStateLoop() { if err := index.StoreInternalState(internalState); err != nil { glog.Error("storeInternalStateLoop ", errors.ErrorStack(err)) } + if lastAppInfo.Add(logAppInfoPeriod).Before(time.Now()) { + glog.Info(index.GetMemoryStats()) + if err := blockbookAppInfoMetric(index, chain, txCache, internalState, metrics); err != nil { + glog.Error("blockbookAppInfoMetric ", err) + } + lastAppInfo = time.Now() + } }) glog.Info("storeInternalStateLoop stopped") } @@ -495,3 +545,9 @@ func printResult(txid string, vout uint32, isOutput bool) error { glog.Info(txid, vout, isOutput) return nil } + +func normalizeName(s string) string { + s = strings.ToLower(s) + s = strings.Replace(s, " ", "-", -1) + return s +} diff --git a/build/docker/bin/Dockerfile b/build/docker/bin/Dockerfile index 89047bb3..b957ad16 100644 --- a/build/docker/bin/Dockerfile +++ b/build/docker/bin/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update && \ apt-get clean ENV GOLANG_VERSION=go1.10.linux-amd64 +ENV ROCKSDB_VERSION=rocksdb-5.14.3 ENV GOPATH=/go ENV PATH=$PATH:$GOPATH/bin ENV CGO_CFLAGS="-I/opt/rocksdb/include" @@ -26,7 +27,7 @@ RUN echo -n "GO version: " && go version RUN echo -n "GOPATH: " && echo $GOPATH # install rocksdb -RUN cd /opt && git clone https://github.com/facebook/rocksdb.git +RUN cd /opt && git clone -b $ROCKSDB_VERSION --depth 1 https://github.com/facebook/rocksdb.git RUN cd /opt/rocksdb && CFLAGS=-fPIC CXXFLAGS=-fPIC make -j 4 release RUN strip /opt/rocksdb/ldb /opt/rocksdb/sst_dump && \ cp /opt/rocksdb/ldb /opt/rocksdb/sst_dump /build diff --git a/build/docker/bin/Makefile b/build/docker/bin/Makefile index 2043a202..f972f896 100644 --- a/build/docker/bin/Makefile +++ b/build/docker/bin/Makefile @@ -4,17 +4,20 @@ VERSION ?= devel GITCOMMIT = $(shell cd /src && git describe --tags --always --dirty) BUILDTIME = $(shell date --iso-8601=seconds) LDFLAGS := -X blockbook/common.version=$(VERSION) -X blockbook/common.gitcommit=$(GITCOMMIT) -X blockbook/common.buildtime=$(BUILDTIME) +BLOCKBOOK_SRC := $(GOPATH)/src/blockbook ARGS ?= +export BLOCKBOOK_SRC + all: build tools build: prepare-sources - cd $(GOPATH)/src/blockbook && go build -o $(CURDIR)/blockbook -ldflags="-s -w $(LDFLAGS)" $(ARGS) + cd $(BLOCKBOOK_SRC) && go build -o $(CURDIR)/blockbook -ldflags="-s -w $(LDFLAGS)" $(ARGS) cp $(CURDIR)/blockbook /out/blockbook chown $(PACKAGER) /out/blockbook build-debug: prepare-sources - cd $(GOPATH)/src/blockbook && go build -o $(CURDIR)/blockbook -ldflags="$(LDFLAGS)" $(ARGS) + cd $(BLOCKBOOK_SRC) && go build -o $(CURDIR)/blockbook -ldflags="$(LDFLAGS)" $(ARGS) cp $(CURDIR)/blockbook /out/blockbook chown $(PACKAGER) /out/blockbook @@ -23,20 +26,20 @@ tools: chown $(PACKAGER) /out/{ldb,sst_dump} test: prepare-sources - cd $(GOPATH)/src/blockbook && go test -tags unittest `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) + cd $(BLOCKBOOK_SRC) && go test -tags unittest `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) test-all: prepare-sources - cd $(GOPATH)/src/blockbook && go test -tags 'unittest integration' `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) + cd $(BLOCKBOOK_SRC) && go test -tags 'unittest integration' `go list ./... | grep -v '^blockbook/contrib'` $(ARGS) prepare-sources: @ [ -n "`ls /src 2> /dev/null`" ] || (echo "/src doesn't exist or is empty" 1>&2 && exit 1) - [ -d $(GOPATH)/src/blockbook ] || cp -r /src $(GOPATH)/src/blockbook + [ -d $(BLOCKBOOK_SRC) ] || cp -r /src $(BLOCKBOOK_SRC) $(MAKE) prepare-vendor prepare-vendor: @ if [ "$(UPDATE_VENDOR)" -eq 1 ]; then \ echo "Updating vendor"; \ - rm -rf $(GOPATH)/src/blockbook/vendor && cd $(GOPATH)/src/blockbook && dep ensure -vendor-only ; \ + rm -rf $(BLOCKBOOK_SRC)/vendor && cd $(BLOCKBOOK_SRC) && dep ensure -vendor-only ; \ else \ echo "Update of vendor not demanded, keeping version from src" ; \ fi diff --git a/build/docker/deb/gpg-keys/bcash-shammah.chancellor.asc b/build/docker/deb/gpg-keys/bcash-shammah.chancellor.asc new file mode 100644 index 00000000..9f7d73d2 --- /dev/null +++ b/build/docker/deb/gpg-keys/bcash-shammah.chancellor.asc @@ -0,0 +1,98 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFpZENYBEACgAfjxYOe3IG0/IUmrpTExiWFvBsVAI2qDn5F11vuimbr1YMGX +PqcmFGWrYZM/zEQCO6xNZEIQ9rh79N15hXfLc9dlLx8PjuxPHNKnPDprpdsYpa37 +Dw0ufHpv+av2qqwG7o9zUan2a+QqsWXjCWhdJVNwYOFY9uRBKqkYMf+r0VOFGPIX +Pjb0n5TXdpI0FFvydeFb+HABnYqbrPcEAQQETpVukQZQtmE8UBDueO+Hz0RvJAw+ +xz79ks4muyPf4MGoKOu6xWM2mViXeP1AN+Mi3nV3EUVPKLxS2EyByrgWEh8iIdgj +vu2fRSaHSuhJWbVSkMnAPJsC3bQE/aZZ8PtaLvgVUH/0euVoKHmxIg8bj1RzNvYb +VZ2VL8Sp8xHIptkTwhgj+spi88BDI1xkBXaHt0ZlddyhgMtK7dl75mn6d+UsGQsT +4dovbtRWRFU1hww7utgtgJniwOYMzhLdY6+OZ8CSA/uB1bTGRigbSd3UtTEwHnQi +xEkHzuF4IoIbaIRLluCASs331vwk+cCZoXD8hdNQdVOwI/4Ss16RAdMFa+DGbBKb +OUpnHBx9wmGH/nq1rfBZD1r/sE7X3LhDeQfqHrRGZOFTAF3CBsiZr7SDAJMCPys6 +7y4KQtC+Kfyku+iDNsnObbCXc86qIrYEOspwMqrZgT3FmA0H+cX9EtoE8QARAQAB +tEhTaGFtbWFoIENoYW5jZWxsb3IgKEJpdGNvaW4gQUJDIGtleSkgPHNoYW1tYWgu +Y2hhbmNlbGxvckBiaXRjb2luYWJjLm9yZz6JAhwEEAEIAAYFAlsWzFAACgkQjVUt +ITLQxfZMCBAAmWotQ7ow3WIEdnBYUxZMMdGxU0WRz13lN+4YnwxAeKoBcpNwFhSz +0wGZAAVzm/hH2aLNMY/Xqt4vq2vuHC1ovOrRIL7Wk0msT/enlyPmK1j+r2zNz1Qk +92ybAfuZ2r/8zO6M1NXOrhFEOmBn6lPLQu79P6T0wGfQue3O0XPx53+WYKmNoyq7 +DQRphmBbRJikdz5TP1ffEUZ+AL901Q/b3gJ4fLpowaJ44C4J+cH++OoLY2lQFPVR +v2JyVa8SeUFfB19ki0m0r9gYEm7RDHlwcLPKwuM45armV6JTE+PW31XZqsE3Xw5h +vGGOT9bY+KoMfgplYJYr9XH1qNgUBYJZ91BAs0vuqieIoMsMHl5h+ii/MDz6eJ9y +5mwR1BKuhS0oIgvoviVU/GhEpo0GQqhsaOejnNl1/kYFpHzosFSf+9EmLWHgicKO +zwumIwsi+AjSI97pg8K8YaxkaA/uagHkutQ1oFgt/tAd6eOGnyNDCBBRSbYVuFJV +YoiRG8IaIHZkUGmrHqDeZd2Qk5YwIG7tLAX9v0cgMlDLFLx1B8LQSY9wH06zjV1a +mY69q3RzQ3mvKCf2QidB5o1FA/CV9Sn0nTvN0UKyYIXNr/KeoyCZkBuge+hD5au/ +OBrtCjGTzq6496hJbwQR83O6S3r4t4KBz2isQuoksrdxwR0cHg65hBqJAhwEEAEI +AAYFAls6O2kACgkQ5EhjVueoHSwGlA/6ApGOSYwQlppEgmwrN2BZLEmfN8pfn4ES +ymYoFKn5/4R2A11Xp5w2z9EE9JW7/yLnoar0jR/Hw16TPceiVoyRD1r5MCEeMDmj +TymDJF4p5ZeiWqkFq1CAvEdCsNB952O2F5PmqGnIm0SW7H8h49Hy1c1Cpa+d0xgg +eQkoEqtihxWabMJ7UW1r6T77dEDoPyAcP2Yi8Yv/e0Eaayv58gxB7ieGi+NsAXLL +LiGkRXhP+FHpBS9yLnxqLEPcDfivMaL3CGocJziA9GlZgSQGUjBkaVoI+Ynd9JTw +C30XususiGthXrqF5e56PIVubOhrxqAlKoxahx/kedRfWPlD/MApNxaokgmDqBpi +E2YLPzaY/RKek/AC5UAgw97yiDactt+UICcHbtG/tznQ6ECO8llI7+DSAWAKCNi1 +iCPZmdjGqv+j/S5xR3X3EdB0LoXpqE6HJE2E1+mUN/QCsz/6u/zmw1Zj5x9niPOp +g5PiIPDTLrhropMKMmvxdfKfKWOXD+I9b/tKldWi7Dz8MPlvXpkEqeA8HKRK8Ek7 +bq9iLdqGFTP/0T+nU77baOmMMSHsoimIIZRysg04n8WR6A4AAOACX/BoB6agxLn8 +1N//xMTqUA0Aror9dLNXl0iCbUshnFkx1wiq93rdZAn6nCI0wlFtg2yhh/dWo1sZ +caGNIgGdxn2JAhwEEAEIAAYFAls7ja0ACgkQXXkiu9ZJxKdf1RAArXapCE/zxWpd +f9cNOClRBLXZlj7FiW7Mf63DgdO0ae4gK4sL812bSwEH4KeuQxkugYsOcH6oezWh +mXxJ8lHOeCFbW+a6VBx7kclUweEOpFtcfrJU1bgLywGM6FtWNn1m1mAMKCrqDgPQ +U2L6UnhY+H3QgRqqYo1nzRsfpqnAxw+7nsquD4ziQS/Mumbj2C0Gjg5ScjkfAidk +okp5jdoIEbanjKQ8zuSRWvomGeT6UF1AmPA3bNtN3ROD9co4H8fKeFKamrCzVZul ++eu5/Y+cCvQ5PEHZoflZYwa3qdTgQJhIlzBmGIipYstHVPfyeFKN9JYXv01FNvDT +h1ebvDzgvaBB/lln3CxWHSEkv7jTu902ljUO9iuPdd0tX89FN1FlLCqeXXoT9Qfi +NOaRJzNDkqdHl53VrhuNH5mZWXXAuD5hkZo1I5A90YuOLOQ5wnl/8MbDS4VW8tpK +HWqn5b0GFacB2xsSphYpletkQnwQWFy352NBveDf68E9rGA9WS5B7cVvnSiNZWNT +5hzE5krS6VZjSIo062PVySTWAqgCy1OWdEPZXo71BpKHV3AgBTAaoPj6EWDFaqS3 +4/5s4odJBUgtceEg4naXFp5GdrrixzInLwjNu1FdpU3NM1thelWzErYi8k8OMVKy +QroQJ+ObZ3PIKBaLgAyOTJq6TZqxa/2JAjcEEwEIACsFAlpZENYJEH05WMREJ2dK +AhsDBQkHhM4ABQsJCAcCBhUKCQgLAgQWAgMBAABVdw/+Jc4K7/8rcWawTG7u2jQG +C3un44dHrVwFxft1IQWbc1ZYx7xvTFtAyOwatwL+PnHjJlMK6ODztJnNHS95CqPS +9dkUztrlQF+j4gmc6/7h0Ew3M56N9tVOXP/3Un+kPgGLdWvVYf0YZsy0r8BO7vQb +PXk2coKiWwfony55DDt9RaxLNlWukKq3dg+vejZzzucfKuZIlUcQDRJO4rCXxMBv +Oo/BtgfOXACjzi4O8aF2TNzEKyXLp8KqcB8j2knu2PIJidr8fwzyBwe3yF8tox3x +4nvSn5IYvvvP4EG8fhsGVBEc+EwZJnbrk5afRBnM/uYW+e+/7vBQwlZyvFOr/zMJ +rhbtcWwW8VkFhvbwc40K7qsiJOxfzXMfGVPtutbzhz8sRivija03LrE0ysgesoM4 +LpxsHDhwScwTTupZ8qsvoNDqgyuJvEUeNXKv1fdY9IVtKgJuU5JHcLOyhUB8/xkd +2qle6fKQK1k776HhdmseU6Ba3cbe3zZYJfYGnoHXttfOZ5QhKNlFhyCdmv7CImP3 +KFYKfYcbx+CVhGJpm5YEYk3WW2FyuBx80WrEbztGcvqZzPQmNPAhvRkVErNoxA/A +KCuLTNrz0jivgBny6JKGdY8drXRafovBroJcLGGFfTVjgZouUis1VlZX4Bo4Twcq +Pc5AK4OXCa/WK1IjXTSblDWJAlQEEwEIAD4WIQR6VaRPOjI5gnyKWU59OVjERCdn +SgUCWlkQ1gIbAwUJB4TOAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRB9OVjE +RCdnSmNWD/47fhdIy2gdHKl+BqjUezcmBs5eR5V8/pz6kbT1+cDFHZ+ImcK+jf8I +8oQPeFWecBF1ZERoLaIRD6UwIVs6z7yAGYlCeweitWf7cfgvIvzP9b/sLweAfwD8 +fNfKTQuw3wx6Yr66+y42lM5dasu+WyzBYLQhQI6q+WgsRL5j7IuNFTdtAc/gNY5c +GlEPXMl9RCjJlCeyo9UNvMeL61UrkPUMVOOqH4Bl8yatPilu0vyGtYogp+XTbmEk +ZJ1BWhL4Y8VkVdBhX9LhYZ+klI6UjNxnZEnVwTwVhZCMXBgi6t4X8EJqeVa5/gb5 +LQ7L261z3y3wzyKZ9+wTjjihXaHX+5n7b1oTZF08+1cY9qe1tDIWfsDY5HjoJb15 +MSqy5EdTJsH+h2JlzMD3t57pTCmGXPmA8mdec78q5DA2PlCdc0qewOTDJ8d56J/x +4PrSZ8TZDRIKA3fflRfWwKrAmB+xwR4cvAXbNC1PxC4BvlVOFKBseDHv7VwWJvnH +0Gnc6hdHdzJJYtTQBglOH6CifHBnksc6/pnFedQ8yWRKXFY3rsaqpzkiiqCPl9jG +QCPWAEvehT6v4iFFiQkZ9OTjBVmT9Ea6BvhpCplpgPo+AFKS3L1mMZYf8RIHazdo +fQof6w/oMxmwQWT/k3O0Z4JRNW0p2eyB8RS3tK3YT2zY3y4TSyYqY7kCDQRaWRDW +ARAA0zg0VamqvOmzSg+viNRFwPmetdPNmiKq7feDail0r8vtfAIKuXcjR6EF0FEB +w9eWlzbX0Eu6c/MVPRycj9Ka+7KqZbEAlOOC97gzA2tcRar8aEM1ZXDBzyr3ZbbK +aH6/XPYm+oxnYxwlMY99/LpiTcOkQyc4yJCXSC8GV9frQhVOGPI7gPca7DjroGdb +as/uTiyPNp4xq0JTI/xWYLRtH+dTEfH2/vTpoXku0itdfR4uoCPFzOYCegfObIOX +3UsUvkoNKWBoQGc14YarTMAjagDDUBhnhJ84b/ftOWTvyYyq/QR4oJdGUkTp854Z +4r2J8cWzW2tpuALxUrHs4AcTmW3cD37B7j3lhzpfA3NGLYwDBQfQkFMwtRIxskf3 +EG7Ednq5fXJWCekIYSji8bDaGu/A5exuj5H0Mg3DKUcVQ+p2bRh6fjKZwG6gEQWU +rxu6RsjFYg6KuLUb0q0Y0Vt/vvLASkY4FoIxjj0zGR+KBlg8yJsU44HdiI40LRPy +2juYgGAhL2j53Yrr+ID0ZFbe9TpQePcBfD5lEWxVH9aBjjfEz1TUYguNQXo3mVrO +OWZek6f9Uh5Yt+WSPSrZfzbRsROcv3OOC83lSYcL2ugiFjt9oGXZBYhfYNLCKDpW +10nGVe0oVSjIA53QIdhQ233WPvwi6ur5Fa8j5OvaIWnwpecAEQEAAYkCPAQYAQgA +JhYhBHpVpE86MjmCfIpZTn05WMREJ2dKBQJaWRDWAhsMBQkHhM4AAAoJEH05WMRE +J2dK5VMP/2IIPi707oF3/iflM3JvQukjmGNLpm3sVrx0baSODW2IfSNQeMWYXvoa +xgaKrMG+WGzr1P/D1jv+HD8gxjakTvZYooJLHF/9nmZE1bD/p0BGFpoRuvX9L/J9 +Yqu32S89lYqnvyXY3c6Aqt/sJj1SHCPdFWMT/kcYjGHeBaX2Ub5+zVm4THESwssv +GsT37N6Wzz0P2wI1rSRu29evKbRTebRIZXaSWiVlx4P7Y6CNFNptzkH7oqf6JmkB +qaQZuOZ3dhe6Z9HetOFN9D5X0ju3DgUITEmF6BYekGBEv/HVJs6rbTb9us8sV4Vu +WvmTYCCIn/fOnmFD6WomjVeX1E6DJ9YvRtCPtau0JcxxBy1L3/3IS2fEf3zpTK6O +aX2GnpijmLFweDGD7CocaEmfw3xUvwgDEpIIvaMNnlciujZFd+8MMJ/JeQ+l5xuw +yWcUciwnWIoCQBOTl3u0u/mWeQsHQr81pirknugjJ8OCPyMUbtsNm9aAySFj6lra +O18EF0uVo+oYDulM+2InSjEJQLYj392XOG4sWkO0bPWJ6FG8VFC2MH09bnsU7dyH +mN6ZWn6JZI3uPY3UbRsXTnVws4OG4bBCpuCe0mJ35gWGAqj0ZDm8SA5Utx4j2oEl +h0HChP/lLDxJ5t5PfBVyudK+xewFFx/cX+4f/n7GZp2+wvduWrs7 +=Grfn +-----END PGP PUBLIC KEY BLOCK----- diff --git a/build/templates/backend/debian/service b/build/templates/backend/debian/service index b494c22c..54473b3b 100644 --- a/build/templates/backend/debian/service +++ b/build/templates/backend/debian/service @@ -7,6 +7,7 @@ After=network.target ExecStart={{template "Backend.ExecCommandTemplate" .}} User={{.Backend.SystemUser}} Restart=on-failure +TimeoutStopSec=300 WorkingDirectory={{.Env.BackendInstallPath}}/{{.Coin.Alias}} {{if eq .Backend.ServiceType "forking" -}} Type=forking diff --git a/build/templates/blockbook/debian/service b/build/templates/blockbook/debian/service index 5bf291bc..e3541215 100644 --- a/build/templates/blockbook/debian/service +++ b/build/templates/blockbook/debian/service @@ -9,6 +9,7 @@ ExecStart={{.Env.BlockbookInstallPath}}/{{.Coin.Alias}}/bin/blockbook -blockchai User={{.Blockbook.SystemUser}} Type=simple Restart=on-failure +TimeoutStopSec=300 WorkingDirectory={{.Env.BlockbookInstallPath}}/{{.Coin.Alias}} # Resource limits diff --git a/build/templates/generate.go b/build/templates/generate.go index 20d2f893..6c44ee7b 100644 --- a/build/templates/generate.go +++ b/build/templates/generate.go @@ -1,126 +1,19 @@ package main import ( - "encoding/json" + "blockbook/build/tools" "fmt" - "io" "os" "path/filepath" "strings" - "text/template" - "time" ) const ( - configsDir = "configs" - inputDir = "build/templates" - outputDir = "build/pkg-defs" + configsDir = "configs" + templateDir = "build/templates" + outputDir = "build/pkg-defs" ) -type Config struct { - Meta struct { - BuildDatetime string // generated field - PackageMaintainer string `json:"package_maintainer"` - PackageMaintainerEmail string `json:"package_maintainer_email"` - } - Env struct { - Version string `json:"version"` - BackendInstallPath string `json:"backend_install_path"` - BackendDataPath string `json:"backend_data_path"` - BlockbookInstallPath string `json:"blockbook_install_path"` - BlockbookDataPath string `json:"blockbook_data_path"` - } `json:"env"` - Coin struct { - Name string `json:"name"` - Shortcut string `json:"shortcut"` - Label string `json:"label"` - Alias string `json:"alias"` - } `json:"coin"` - Ports struct { - BackendRPC int `json:"backend_rpc"` - BackendMessageQueue int `json:"backend_message_queue"` - BlockbookInternal int `json:"blockbook_internal"` - BlockbookPublic int `json:"blockbook_public"` - } `json:"ports"` - IPC struct { - RPCURLTemplate string `json:"rpc_url_template"` - RPCUser string `json:"rpc_user"` - RPCPass string `json:"rpc_pass"` - RPCTimeout int `json:"rpc_timeout"` - MessageQueueBindingTemplate string `json:"message_queue_binding_template"` - } `json:"ipc"` - Backend struct { - PackageName string `json:"package_name"` - PackageRevision string `json:"package_revision"` - SystemUser string `json:"system_user"` - Version string `json:"version"` - BinaryURL string `json:"binary_url"` - VerificationType string `json:"verification_type"` - VerificationSource string `json:"verification_source"` - ExtractCommand string `json:"extract_command"` - ExcludeFiles []string `json:"exclude_files"` - ExecCommandTemplate string `json:"exec_command_template"` - LogrotateFilesTemplate string `json:"logrotate_files_template"` - PostinstScriptTemplate string `json:"postinst_script_template"` - ServiceType string `json:"service_type"` - ServiceAdditionalParamsTemplate string `json:"service_additional_params_template"` - ProtectMemory bool `json:"protect_memory"` - Mainnet bool `json:"mainnet"` - ConfigFile string `json:"config_file"` - AdditionalParams interface{} `json:"additional_params"` - } `json:"backend"` - Blockbook struct { - PackageName string `json:"package_name"` - SystemUser string `json:"system_user"` - InternalBindingTemplate string `json:"internal_binding_template"` - PublicBindingTemplate string `json:"public_binding_template"` - ExplorerURL string `json:"explorer_url"` - AdditionalParams string `json:"additional_params"` - BlockChain struct { - Parse bool `json:"parse"` - Subversion string `json:"subversion"` - AddressFormat string `json:"address_format"` - MempoolWorkers int `json:"mempool_workers"` - MempoolSubWorkers int `json:"mempool_sub_workers"` - BlockAddressesToKeep int `json:"block_addresses_to_keep"` - AdditionalParams map[string]json.RawMessage `json:"additional_params"` - } `json:"block_chain"` - } `json:"blockbook"` -} - -func jsonToString(msg json.RawMessage) (string, error) { - d, err := msg.MarshalJSON() - if err != nil { - return "", err - } - return string(d), nil -} - -func (c *Config) ParseTemplate() *template.Template { - templates := map[string]string{ - "IPC.RPCURLTemplate": c.IPC.RPCURLTemplate, - "IPC.MessageQueueBindingTemplate": c.IPC.MessageQueueBindingTemplate, - "Backend.ExecCommandTemplate": c.Backend.ExecCommandTemplate, - "Backend.LogrotateFilesTemplate": c.Backend.LogrotateFilesTemplate, - "Backend.PostinstScriptTemplate": c.Backend.PostinstScriptTemplate, - "Backend.ServiceAdditionalParamsTemplate": c.Backend.ServiceAdditionalParamsTemplate, - "Blockbook.InternalBindingTemplate": c.Blockbook.InternalBindingTemplate, - "Blockbook.PublicBindingTemplate": c.Blockbook.PublicBindingTemplate, - } - - funcMap := template.FuncMap{ - "jsonToString": jsonToString, - } - - t := template.New("").Funcs(funcMap) - - for name, def := range templates { - t = template.Must(t.Parse(fmt.Sprintf(`{{define "%s"}}%s{{end}}`, name, def))) - } - - return t -} - func main() { if len(os.Args) < 2 { var coins []string @@ -136,171 +29,12 @@ func main() { } coin := os.Args[1] - config := loadConfig(coin) - generatePackageDefinitions(config) + config, err := build.LoadConfig(configsDir, coin) + if err == nil { + err = build.GeneratePackageDefinitions(config, templateDir, outputDir) + } + if err != nil { + panic(err) + } fmt.Printf("Package files for %v generated to %v\n", coin, outputDir) } - -func loadConfig(coin string) *Config { - config := new(Config) - - f, err := os.Open(filepath.Join(configsDir, "coins", coin+".json")) - if err != nil { - panic(err) - } - d := json.NewDecoder(f) - err = d.Decode(config) - if err != nil { - panic(err) - } - - f, err = os.Open(filepath.Join(configsDir, "environ.json")) - if err != nil { - panic(err) - } - d = json.NewDecoder(f) - err = d.Decode(&config.Env) - if err != nil { - panic(err) - } - - config.Meta.BuildDatetime = time.Now().Format("Mon, 02 Jan 2006 15:04:05 -0700") - - if !isEmpty(config, "backend") { - switch config.Backend.ServiceType { - case "forking": - case "simple": - default: - panic("Invalid service type: " + config.Backend.ServiceType) - } - - switch config.Backend.VerificationType { - case "": - case "gpg": - case "sha256": - case "gpg-sha256": - default: - panic("Invalid verification type: " + config.Backend.VerificationType) - } - } - - return config -} - -func isEmpty(config *Config, target string) bool { - switch target { - case "backend": - return config.Backend.PackageName == "" - case "blockbook": - return config.Blockbook.PackageName == "" - default: - panic("Invalid target name: " + target) - } -} - -func generatePackageDefinitions(config *Config) { - templ := config.ParseTemplate() - - makeOutputDir(outputDir) - - for _, subdir := range []string{"backend", "blockbook"} { - if isEmpty(config, subdir) { - continue - } - - root := filepath.Join(inputDir, subdir) - - err := os.Mkdir(filepath.Join(outputDir, subdir), 0755) - if err != nil { - panic(err) - } - - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("%s: %s", path, err) - } - - if path == root { - return nil - } - if filepath.Base(path)[0] == '.' { - return nil - } - - subpath := path[len(root)-len(subdir):] - - if info.IsDir() { - err = os.Mkdir(filepath.Join(outputDir, subpath), info.Mode()) - if err != nil { - return fmt.Errorf("%s: %s", path, err) - } - return nil - } - - t := template.Must(templ.Clone()) - t = template.Must(t.ParseFiles(path)) - - err = writeTemplate(filepath.Join(outputDir, subpath), info, t, config) - if err != nil { - return fmt.Errorf("%s: %s", path, err) - } - - return nil - }) - if err != nil { - panic(err) - } - } - - if !isEmpty(config, "backend") { - writeBackendConfigFile(config) - } -} - -func makeOutputDir(path string) { - err := os.RemoveAll(path) - if err == nil { - err = os.Mkdir(path, 0755) - } - if err != nil { - panic(err) - } -} - -func writeTemplate(path string, info os.FileInfo, templ *template.Template, config *Config) error { - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, info.Mode()) - if err != nil { - return err - } - defer f.Close() - - err = templ.ExecuteTemplate(f, "main", config) - if err != nil { - return err - } - - return nil -} - -func writeBackendConfigFile(config *Config) { - out, err := os.OpenFile(filepath.Join(outputDir, "backend/backend.conf"), os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - panic(err) - } - defer out.Close() - - if config.Backend.ConfigFile == "" { - return - } else { - in, err := os.Open(filepath.Join(outputDir, "backend/config", config.Backend.ConfigFile)) - if err != nil { - panic(err) - } - defer in.Close() - - _, err = io.Copy(out, in) - if err != nil { - panic(err) - } - } -} diff --git a/build/tools/templates.go b/build/tools/templates.go new file mode 100644 index 00000000..c2e34157 --- /dev/null +++ b/build/tools/templates.go @@ -0,0 +1,288 @@ +package build + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "text/template" + "time" +) + +type Config struct { + Meta struct { + BuildDatetime string // generated field + PackageMaintainer string `json:"package_maintainer"` + PackageMaintainerEmail string `json:"package_maintainer_email"` + } + Env struct { + Version string `json:"version"` + BackendInstallPath string `json:"backend_install_path"` + BackendDataPath string `json:"backend_data_path"` + BlockbookInstallPath string `json:"blockbook_install_path"` + BlockbookDataPath string `json:"blockbook_data_path"` + } `json:"env"` + Coin struct { + Name string `json:"name"` + Shortcut string `json:"shortcut"` + Label string `json:"label"` + Alias string `json:"alias"` + } `json:"coin"` + Ports struct { + BackendRPC int `json:"backend_rpc"` + BackendMessageQueue int `json:"backend_message_queue"` + BlockbookInternal int `json:"blockbook_internal"` + BlockbookPublic int `json:"blockbook_public"` + } `json:"ports"` + IPC struct { + RPCURLTemplate string `json:"rpc_url_template"` + RPCUser string `json:"rpc_user"` + RPCPass string `json:"rpc_pass"` + RPCTimeout int `json:"rpc_timeout"` + MessageQueueBindingTemplate string `json:"message_queue_binding_template"` + } `json:"ipc"` + Backend struct { + PackageName string `json:"package_name"` + PackageRevision string `json:"package_revision"` + SystemUser string `json:"system_user"` + Version string `json:"version"` + BinaryURL string `json:"binary_url"` + VerificationType string `json:"verification_type"` + VerificationSource string `json:"verification_source"` + ExtractCommand string `json:"extract_command"` + ExcludeFiles []string `json:"exclude_files"` + ExecCommandTemplate string `json:"exec_command_template"` + LogrotateFilesTemplate string `json:"logrotate_files_template"` + PostinstScriptTemplate string `json:"postinst_script_template"` + ServiceType string `json:"service_type"` + ServiceAdditionalParamsTemplate string `json:"service_additional_params_template"` + ProtectMemory bool `json:"protect_memory"` + Mainnet bool `json:"mainnet"` + ConfigFile string `json:"config_file"` + AdditionalParams interface{} `json:"additional_params"` + } `json:"backend"` + Blockbook struct { + PackageName string `json:"package_name"` + SystemUser string `json:"system_user"` + InternalBindingTemplate string `json:"internal_binding_template"` + PublicBindingTemplate string `json:"public_binding_template"` + ExplorerURL string `json:"explorer_url"` + AdditionalParams string `json:"additional_params"` + BlockChain struct { + Parse bool `json:"parse"` + Subversion string `json:"subversion"` + AddressFormat string `json:"address_format"` + MempoolWorkers int `json:"mempool_workers"` + MempoolSubWorkers int `json:"mempool_sub_workers"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` + AdditionalParams map[string]json.RawMessage `json:"additional_params"` + } `json:"block_chain"` + } `json:"blockbook"` + IntegrationTests map[string][]string `json:"integration_tests"` +} + +func jsonToString(msg json.RawMessage) (string, error) { + d, err := msg.MarshalJSON() + if err != nil { + return "", err + } + return string(d), nil +} + +func (c *Config) ParseTemplate() *template.Template { + templates := map[string]string{ + "IPC.RPCURLTemplate": c.IPC.RPCURLTemplate, + "IPC.MessageQueueBindingTemplate": c.IPC.MessageQueueBindingTemplate, + "Backend.ExecCommandTemplate": c.Backend.ExecCommandTemplate, + "Backend.LogrotateFilesTemplate": c.Backend.LogrotateFilesTemplate, + "Backend.PostinstScriptTemplate": c.Backend.PostinstScriptTemplate, + "Backend.ServiceAdditionalParamsTemplate": c.Backend.ServiceAdditionalParamsTemplate, + "Blockbook.InternalBindingTemplate": c.Blockbook.InternalBindingTemplate, + "Blockbook.PublicBindingTemplate": c.Blockbook.PublicBindingTemplate, + } + + funcMap := template.FuncMap{ + "jsonToString": jsonToString, + } + + t := template.New("").Funcs(funcMap) + + for name, def := range templates { + t = template.Must(t.Parse(fmt.Sprintf(`{{define "%s"}}%s{{end}}`, name, def))) + } + + return t +} + +func LoadConfig(configsDir, coin string) (*Config, error) { + config := new(Config) + + f, err := os.Open(filepath.Join(configsDir, "coins", coin+".json")) + if err != nil { + return nil, err + } + d := json.NewDecoder(f) + err = d.Decode(config) + if err != nil { + return nil, err + } + + f, err = os.Open(filepath.Join(configsDir, "environ.json")) + if err != nil { + return nil, err + } + d = json.NewDecoder(f) + err = d.Decode(&config.Env) + if err != nil { + return nil, err + } + + config.Meta.BuildDatetime = time.Now().Format("Mon, 02 Jan 2006 15:04:05 -0700") + + if !isEmpty(config, "backend") { + switch config.Backend.ServiceType { + case "forking": + case "simple": + default: + return nil, fmt.Errorf("Invalid service type: %s", config.Backend.ServiceType) + } + + switch config.Backend.VerificationType { + case "": + case "gpg": + case "sha256": + case "gpg-sha256": + default: + return nil, fmt.Errorf("Invalid verification type: %s", config.Backend.VerificationType) + } + } + + return config, nil +} + +func isEmpty(config *Config, target string) bool { + switch target { + case "backend": + return config.Backend.PackageName == "" + case "blockbook": + return config.Blockbook.PackageName == "" + default: + panic("Invalid target name: " + target) + } +} + +func GeneratePackageDefinitions(config *Config, templateDir, outputDir string) error { + templ := config.ParseTemplate() + + err := makeOutputDir(outputDir) + if err != nil { + return err + } + + for _, subdir := range []string{"backend", "blockbook"} { + if isEmpty(config, subdir) { + continue + } + + root := filepath.Join(templateDir, subdir) + + err = os.Mkdir(filepath.Join(outputDir, subdir), 0755) + if err != nil { + return err + } + + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("%s: %s", path, err) + } + + if path == root { + return nil + } + if filepath.Base(path)[0] == '.' { + return nil + } + + subpath := path[len(root)-len(subdir):] + + if info.IsDir() { + err = os.Mkdir(filepath.Join(outputDir, subpath), info.Mode()) + if err != nil { + return fmt.Errorf("%s: %s", path, err) + } + return nil + } + + t := template.Must(templ.Clone()) + t = template.Must(t.ParseFiles(path)) + + err = writeTemplate(filepath.Join(outputDir, subpath), info, t, config) + if err != nil { + return fmt.Errorf("%s: %s", path, err) + } + + return nil + }) + if err != nil { + return err + } + } + + if !isEmpty(config, "backend") { + err = writeBackendConfigFile(config, outputDir) + if err != nil { + return err + } + } + + return nil +} + +func makeOutputDir(path string) error { + err := os.RemoveAll(path) + if err == nil { + err = os.Mkdir(path, 0755) + } + return err +} + +func writeTemplate(path string, info os.FileInfo, templ *template.Template, config *Config) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + defer f.Close() + + err = templ.ExecuteTemplate(f, "main", config) + if err != nil { + return err + } + + return nil +} + +func writeBackendConfigFile(config *Config, outputDir string) error { + out, err := os.OpenFile(filepath.Join(outputDir, "backend/backend.conf"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer out.Close() + + if config.Backend.ConfigFile == "" { + return nil + } else { + in, err := os.Open(filepath.Join(outputDir, "backend/config", config.Backend.ConfigFile)) + if err != nil { + return err + } + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + } + + return nil +} diff --git a/common/internalstate.go b/common/internalstate.go index 8e7b2b61..9479d057 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -11,6 +11,8 @@ const ( DbStateClosed = uint32(iota) // DbStateOpen means db is open or application died without closing the db DbStateOpen + // DbStateInconsistent means db is in inconsistent state and cannot be used + DbStateInconsistent ) // InternalStateColumn contains the data of a db column @@ -35,6 +37,10 @@ type InternalState struct { LastStore time.Time `json:"lastStore"` + // true if application is with flag --sync + SyncMode bool `json:"syncMode"` + + InitialSync bool `json:"initialSync"` IsSynchronized bool `json:"isSynchronized"` BestHeight uint32 `json:"bestHeight"` LastSync time.Time `json:"lastSync"` @@ -62,6 +68,14 @@ func (is *InternalState) FinishedSync(bestHeight uint32) { is.LastSync = time.Now() } +// UpdateBestHeight sets new best height, without changing IsSynchronized flag +func (is *InternalState) UpdateBestHeight(bestHeight uint32) { + is.mux.Lock() + defer is.mux.Unlock() + is.BestHeight = bestHeight + is.LastSync = time.Now() +} + // FinishedSyncNoChange marks end of synchronization in case no index update was necessary, it does not update lastSync time func (is *InternalState) FinishedSyncNoChange() { is.mux.Lock() diff --git a/common/metrics.go b/common/metrics.go index e17cd489..9451180b 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -21,6 +21,7 @@ type Metrics struct { MempoolSize prometheus.Gauge DbColumnRows *prometheus.GaugeVec DbColumnSize *prometheus.GaugeVec + BlockbookAppInfo *prometheus.GaugeVec } type Labels = prometheus.Labels @@ -139,6 +140,14 @@ func GetMetrics(coin string) (*Metrics, error) { }, []string{"column"}, ) + metrics.BlockbookAppInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "blockbook_app_info", + Help: "Information about blockbook and backend application versions", + ConstLabels: Labels{"coin": coin}, + }, + []string{"blockbook_version", "blockbook_commit", "backend_version", "backend_subversion", "backend_protocol_version"}, + ) v := reflect.ValueOf(metrics) for i := 0; i < v.NumField(); i++ { diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index e1bfe518..006488b5 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.17.1", - "binary_url": "https://download.bitcoinabc.org/0.17.1/linux/bitcoin-abc-0.17.1-x86_64-linux-gnu.tar.gz", + "version": "0.18.2", + "binary_url": "https://download.bitcoinabc.org/0.18.2/linux/bitcoin-abc-0.18.2-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "eccf8b61ba0549f6839e586c7dc6fc4bf6d7591ac432aaea8a7df0266b113d27", + "verification_source": "28d8511789a126aff16e256a03288948f2660c3c8cb0a4c809c5a8618a519a16", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -44,12 +44,12 @@ "system_user": "blockbook-bcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://bitcoincash.blockexplorer.com", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, "subversion": "/Bitcoin ABC:0.17.1/", - "address_format": "legacy", + "address_format": "cashaddr", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, @@ -59,5 +59,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index af58f66a..be7fa535 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash-testnet", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.17.1", - "binary_url": "https://download.bitcoinabc.org/0.17.1/linux/bitcoin-abc-0.17.1-x86_64-linux-gnu.tar.gz", + "version": "0.18.2", + "binary_url": "https://download.bitcoinabc.org/0.18.2/linux/bitcoin-abc-0.18.2-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "eccf8b61ba0549f6839e586c7dc6fc4bf6d7591ac432aaea8a7df0266b113d27", + "verification_source": "28d8511789a126aff16e256a03288948f2660c3c8cb0a4c809c5a8618a519a16", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -44,12 +44,12 @@ "system_user": "blockbook-bcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://bitcoincash.blockexplorer.com", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, "subversion": "/Bitcoin ABC:0.17.1/", - "address_format": "legacy", + "address_format": "cashaddr", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, @@ -59,5 +59,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/bgold.json b/configs/coins/bgold.json index 3eb019a6..cc14bdf2 100644 --- a/configs/coins/bgold.json +++ b/configs/coins/bgold.json @@ -22,10 +22,10 @@ "package_name": "backend-bgold", "package_revision": "satoshilabs-1", "system_user": "bgold", - "version": "0.15.1", - "binary_url": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.1/bitcoin-gold-0.15.1-x86_64-linux-gnu.tar.gz", + "version": "0.15.2", + "binary_url": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.2/bitcoin-gold-0.15.2-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.1/SHA256SUMS.asc", + "verification_source": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.2/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -239,11 +239,11 @@ "system_user": "blockbook-bgold", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btg-explorer.trezor.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Bitcoin Gold:0.15.1/", + "subversion": "/Bitcoin Gold:0.15.2/", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index d5435bba..725fb62e 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -22,10 +22,10 @@ "package_name": "backend-bitcoin", "package_revision": "satoshilabs-1", "system_user": "bitcoin", - "version": "0.16.1", - "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.1/bitcoin-0.16.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.1/SHA256SUMS.asc", + "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -47,8 +47,8 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btc-explorer.trezor.io/", - "additional_params": "", + "explorer_url": "", + "additional_params": "-dbcache=1073741824", "block_chain": { "parse": true, "mempool_workers": 8, @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index 3397cb95..8a2ee114 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bitcoin-testnet", "package_revision": "satoshilabs-1", "system_user": "bitcoin", - "version": "0.16.1", - "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.1/bitcoin-0.16.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.1/SHA256SUMS.asc", + "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btc-testnet-explorer.trezor.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/dash.json b/configs/coins/dash.json index d2895671..1c7b77d5 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -22,10 +22,10 @@ "package_name": "backend-dash", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.12.3", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/dashcore-0.12.3.2-x86_64-linux-gnu.tar.gz", + "version": "0.12.3.3", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/dashcore-0.12.3.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" @@ -47,11 +47,11 @@ "system_user": "blockbook-dash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://dash-explorer.trezor.io", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Dash Core:0.12.3.2/", + "subversion": "/Dash Core:0.12.3.3/", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, @@ -61,5 +61,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index 38407e37..98b4ddc6 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-dash-testnet", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.12.3", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/dashcore-0.12.3.2-x86_64-linux-gnu.tar.gz", + "version": "0.12.3.3", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/dashcore-0.12.3.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" @@ -47,11 +47,11 @@ "system_user": "blockbook-dash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://dash-explorer.trezor.io", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Dash Core:0.12.3.2/", + "subversion": "/Dash Core:0.12.3.3/", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, @@ -61,5 +61,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/dogecoin.json b/configs/coins/dogecoin.json index cb9b7a68..a04bda04 100644 --- a/configs/coins/dogecoin.json +++ b/configs/coins/dogecoin.json @@ -63,5 +63,8 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync"] } } diff --git a/configs/coins/ethereum_testnet_ropsten.json b/configs/coins/ethereum_testnet_ropsten.json index 42fe3fd0..30605027 100644 --- a/configs/coins/ethereum_testnet_ropsten.json +++ b/configs/coins/ethereum_testnet_ropsten.json @@ -53,5 +53,8 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/litecoin.json b/configs/coins/litecoin.json index e4541bf9..e2ae20fa 100644 --- a/configs/coins/litecoin.json +++ b/configs/coins/litecoin.json @@ -22,10 +22,10 @@ "package_name": "backend-litecoin", "package_revision": "satoshilabs-1", "system_user": "litecoin", - "version": "0.16.0", - "binary_url": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-linux-signatures.asc", + "verification_source": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-linux-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/litecoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-litecoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://ltc-explorer.trezor.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee"] } } diff --git a/configs/coins/litecoin_testnet.json b/configs/coins/litecoin_testnet.json index 819ae8c0..8eae9a61 100644 --- a/configs/coins/litecoin_testnet.json +++ b/configs/coins/litecoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-litecoin-testnet", "package_revision": "satoshilabs-1", "system_user": "litecoin", - "version": "0.16.0", - "binary_url": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-linux-signatures.asc", + "verification_source": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-linux-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/litecoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-litecoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://ltc-explorer.trezor.io", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/monacoin.json b/configs/coins/monacoin.json index 563b1db6..e3009d80 100644 --- a/configs/coins/monacoin.json +++ b/configs/coins/monacoin.json @@ -22,10 +22,10 @@ "package_name": "backend-monacoin", "package_revision": "satoshilabs-1", "system_user": "monacoin", - "version": "0.15.1", - "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-signatures.asc", + "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/monacoin-qt" @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "wakiyamap", "package_maintainer_email": "wakiyamap@gmail.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee"] } } diff --git a/configs/coins/monacoin_testnet.json b/configs/coins/monacoin_testnet.json index 9e0d06ab..23ac8ce6 100644 --- a/configs/coins/monacoin_testnet.json +++ b/configs/coins/monacoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-monacoin-testnet", "package_revision": "satoshilabs-1", "system_user": "monacoin", - "version": "0.15.1", - "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-signatures.asc", + "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/monacoin-qt" diff --git a/configs/coins/namecoin.json b/configs/coins/namecoin.json index 754ee628..b0f16f65 100644 --- a/configs/coins/namecoin.json +++ b/configs/coins/namecoin.json @@ -22,10 +22,10 @@ "package_name": "backend-namecoin", "package_revision": "satoshilabs-1", "system_user": "namecoin", - "version": "0.13.99", - "binary_url": "https://namecoin.org/files/namecoin-core-0.13.99-name-tab-beta1-notreproduced/namecoin-0.13.99-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://www.namecoin.org/files/namecoin-core-0.16.3/namecoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "294b1106001d6ea2b9d9ee6a655021ef207a24e8f1dec8efd5899728b3849129", + "verification_source": "14ebaaf6f22f69b057a5bcb9b6959548f0a3f1b62cc113f19581d2297044827e", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/namecoin-qt" @@ -54,7 +54,7 @@ "system_user": "blockbook-namecoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://namecha.in/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, @@ -67,5 +67,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/vertcoin.json b/configs/coins/vertcoin.json index d68a0a0f..605d9bfc 100644 --- a/configs/coins/vertcoin.json +++ b/configs/coins/vertcoin.json @@ -22,10 +22,10 @@ "package_name": "backend-vertcoin", "package_revision": "satoshilabs-1", "system_user": "vertcoin", - "version": "0.13.2", - "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip", + "version": "0.13.3", + "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip", "verification_type": "gpg", - "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip.sig", + "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip.sig", "extract_command": "unzip -d backend", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/vertcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -45,7 +45,7 @@ "system_user": "blockbook-vertcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://insight.vertcoin.org", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, @@ -58,5 +58,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/vertcoin_testnet.json b/configs/coins/vertcoin_testnet.json index 314929c3..2f8dfaee 100644 --- a/configs/coins/vertcoin_testnet.json +++ b/configs/coins/vertcoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-vertcoin-testnet", "package_revision": "satoshilabs-1", "system_user": "vertcoin", - "version": "0.13.2", - "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip", + "version": "0.13.3", + "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip", "verification_type": "gpg", - "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip.sig", + "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip.sig", "extract_command": "unzip -d backend", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/vertcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -45,7 +45,7 @@ "system_user": "blockbook-vertcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://insight.vertcoin.org/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index ab65b246..4a6e60ee 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "1.1.1", - "binary_url": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz", + "version": "2.0.0", + "binary_url": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz", "verification_type": "gpg", - "verification_source": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz.asc", + "verification_source": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -47,7 +47,7 @@ "system_user": "blockbook-zcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://zcash.blockexplorer.com/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index c2793c96..f3111de2 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash-testnet", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "1.1.1", - "binary_url": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz", + "version": "2.0.0", + "binary_url": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz", "verification_type": "gpg", - "verification_source": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz.asc", + "verification_source": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -47,7 +47,7 @@ "system_user": "blockbook-zcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://explorer.testnet.z.cash/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, @@ -60,5 +60,9 @@ "meta": { "package_maintainer": "Jakub Matys", "package_maintainer_email": "jakub.matys@satoshilabs.com" + }, + "integration_tests": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", + "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"] } } diff --git a/configs/environ.json b/configs/environ.json index 1f293514..3adf012c 100644 --- a/configs/environ.json +++ b/configs/environ.json @@ -1,5 +1,5 @@ { - "version": "0.0.6", + "version": "0.1.0", "backend_install_path": "/opt/coins/nodes", "backend_data_path": "/opt/coins/data", "blockbook_install_path": "/opt/coins/blockbook", diff --git a/contrib/scripts/generate-port-registry.go b/contrib/scripts/check-and-generate-port-registry.go similarity index 80% rename from contrib/scripts/generate-port-registry.go rename to contrib/scripts/check-and-generate-port-registry.go index 20375eea..94e7c2c4 100755 --- a/contrib/scripts/generate-port-registry.go +++ b/contrib/scripts/check-and-generate-port-registry.go @@ -36,6 +36,67 @@ type Config struct { Ports map[string]uint16 `json:"ports"` } +func checkPorts() int { + ports := make(map[uint16][]string) + status := 0 + + files, err := ioutil.ReadDir(inputDir) + if err != nil { + panic(err) + } + + for _, fi := range files { + if fi.IsDir() || fi.Name()[0] == '.' { + continue + } + + path := filepath.Join(inputDir, fi.Name()) + f, err := os.Open(path) + if err != nil { + panic(fmt.Errorf("%s: %s", path, err)) + } + defer f.Close() + + v := Config{} + d := json.NewDecoder(f) + err = d.Decode(&v) + if err != nil { + panic(fmt.Errorf("%s: json: %s", path, err)) + } + + if _, ok := v.Ports["blockbook_internal"]; !ok { + fmt.Printf("%s: missing blockbook_internal port\n", v.Coin.Name) + status = 1 + } + if _, ok := v.Ports["blockbook_public"]; !ok { + fmt.Printf("%s: missing blockbook_public port\n", v.Coin.Name) + status = 1 + } + if _, ok := v.Ports["backend_rpc"]; !ok { + fmt.Printf("%s: missing backend_rpc port\n", v.Coin.Name) + status = 1 + } + + for _, port := range v.Ports { + if port > 0 { + ports[port] = append(ports[port], v.Coin.Name) + } + } + } + + for port, coins := range ports { + if len(coins) > 1 { + fmt.Printf("port %d: registered by %q\n", port, coins) + status = 1 + } + } + + if status != 0 { + fmt.Println("Got some errors") + } + return status +} + func main() { output := "stdout" if len(os.Args) > 1 { @@ -48,6 +109,11 @@ func main() { } } + status := checkPorts() + if status != 0 { + os.Exit(status) + } + slice, err := loadPortInfo(inputDir) if err != nil { panic(err) diff --git a/contrib/scripts/check-ports.go b/contrib/scripts/check-ports.go deleted file mode 100755 index 1e509de1..00000000 --- a/contrib/scripts/check-ports.go +++ /dev/null @@ -1,83 +0,0 @@ -//usr/bin/go run $0 $@ ; exit -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" -) - -const configDir = "configs/coins" - -type Config struct { - Coin struct { - Name string `json:"name"` - } - Ports map[string]uint16 `json:"ports"` -} - -func main() { - ports := make(map[uint16][]string) - status := 0 - - files, err := ioutil.ReadDir(configDir) - if err != nil { - panic(err) - } - - for _, fi := range files { - if fi.IsDir() || fi.Name()[0] == '.' { - continue - } - - path := filepath.Join(configDir, fi.Name()) - f, err := os.Open(path) - if err != nil { - panic(fmt.Errorf("%s: %s", path, err)) - } - defer f.Close() - - v := Config{} - d := json.NewDecoder(f) - err = d.Decode(&v) - if err != nil { - panic(fmt.Errorf("%s: json: %s", path, err)) - } - - if _, ok := v.Ports["blockbook_internal"]; !ok { - fmt.Printf("%s: missing blockbook_internal port\n", v.Coin.Name) - status = 1 - } - if _, ok := v.Ports["blockbook_public"]; !ok { - fmt.Printf("%s: missing blockbook_public port\n", v.Coin.Name) - status = 1 - } - if _, ok := v.Ports["backend_rpc"]; !ok { - fmt.Printf("%s: missing backend_rpc port\n", v.Coin.Name) - status = 1 - } - - for _, port := range v.Ports { - if port > 0 { - ports[port] = append(ports[port], v.Coin.Name) - } - } - } - - for port, coins := range ports { - if len(coins) > 1 { - fmt.Printf("port %d: registered by %q\n", port, coins) - status = 1 - } - } - - if status == 0 { - fmt.Println("OK") - } else { - fmt.Println("Got some errors") - } - - os.Exit(status) -} diff --git a/db/bulkconnect.go b/db/bulkconnect.go new file mode 100644 index 00000000..60ca7a64 --- /dev/null +++ b/db/bulkconnect.go @@ -0,0 +1,269 @@ +package db + +import ( + "blockbook/bchain" + "time" + + "github.com/golang/glog" + "github.com/tecbot/gorocksdb" +) + +// bulk connect +// in bulk mode the data are cached and stored to db in batches +// it speeds up the import in two ways: +// 1) balances and txAddresses are modified several times during the import, there is a chance that the modifications are done before write to DB +// 2) rocksdb seems to handle better fewer larger batches than continuous stream of smaller batches + +type bulkAddresses struct { + bi BlockInfo + addresses map[string][]outpoint +} + +// BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way +type BulkConnect struct { + d *RocksDB + isUTXO bool + bulkAddresses []bulkAddresses + bulkAddressesCount int + txAddressesMap map[string]*TxAddresses + balances map[string]*AddrBalance + height uint32 +} + +const ( + maxBulkAddresses = 200000 + maxBulkTxAddresses = 500000 + partialStoreAddresses = maxBulkTxAddresses / 10 + maxBulkBalances = 800000 + partialStoreBalances = maxBulkBalances / 10 +) + +// InitBulkConnect initializes bulk connect and switches DB to inconsistent state +func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { + bc := &BulkConnect{ + d: d, + isUTXO: d.chainParser.IsUTXOChain(), + txAddressesMap: make(map[string]*TxAddresses), + balances: make(map[string]*AddrBalance), + } + if err := d.SetInconsistentState(true); err != nil { + return nil, err + } + glog.Info("rocksdb: bulk connect init, db set to inconsistent state") + return bc, nil +} + +func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int, int, error) { + var txm map[string]*TxAddresses + var sp int + if all { + txm = b.txAddressesMap + b.txAddressesMap = make(map[string]*TxAddresses) + } else { + txm = make(map[string]*TxAddresses) + for k, a := range b.txAddressesMap { + // store all completely spent transactions, they will not be modified again + r := true + for _, o := range a.Outputs { + if o.Spent == false { + r = false + break + } + } + if r { + txm[k] = a + delete(b.txAddressesMap, k) + } + } + sp = len(txm) + // store some other random transactions if necessary + if len(txm) < partialStoreAddresses { + for k, a := range b.txAddressesMap { + txm[k] = a + delete(b.txAddressesMap, k) + if len(txm) >= partialStoreAddresses { + break + } + } + } + } + if err := b.d.storeTxAddresses(wb, txm); err != nil { + return 0, 0, err + } + return len(txm), sp, nil +} + +func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) { + defer close(c) + start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + count, sp, err := b.storeTxAddresses(wb, all) + if err != nil { + c <- err + return + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + return + } + glog.Info("rocksdb: height ", b.height, ", stored ", count, " (", sp, " spent) txAddresses, ", len(b.txAddressesMap), " remaining, done in ", time.Since(start)) + c <- nil +} + +func (b *BulkConnect) storeBalances(wb *gorocksdb.WriteBatch, all bool) (int, error) { + var bal map[string]*AddrBalance + if all { + bal = b.balances + b.balances = make(map[string]*AddrBalance) + } else { + bal = make(map[string]*AddrBalance) + // store some random balances + for k, a := range b.balances { + bal[k] = a + delete(b.balances, k) + if len(bal) >= partialStoreBalances { + break + } + } + } + if err := b.d.storeBalances(wb, bal); err != nil { + return 0, err + } + return len(bal), nil +} + +func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) { + defer close(c) + start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + count, err := b.storeBalances(wb, all) + if err != nil { + c <- err + return + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + c <- err + return + } + glog.Info("rocksdb: height ", b.height, ", stored ", count, " balances, ", len(b.balances), " remaining, done in ", time.Since(start)) + c <- nil +} + +func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { + for _, ba := range b.bulkAddresses { + if err := b.d.storeAddresses(wb, ba.bi.Height, ba.addresses); err != nil { + return err + } + if err := b.d.writeHeight(wb, ba.bi.Height, &ba.bi, opInsert); err != nil { + return err + } + } + b.bulkAddressesCount = 0 + b.bulkAddresses = b.bulkAddresses[:0] + return nil +} + +// ConnectBlock connects block in bulk mode +func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { + b.height = block.Height + if !b.isUTXO { + return b.d.ConnectBlock(block) + } + addresses := make(map[string][]outpoint) + if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { + return err + } + var storeAddressesChan, storeBalancesChan chan error + var sa bool + if len(b.txAddressesMap) > maxBulkTxAddresses || len(b.balances) > maxBulkBalances { + sa = true + if len(b.txAddressesMap)+partialStoreAddresses > maxBulkTxAddresses { + storeAddressesChan = make(chan error) + go b.parallelStoreTxAddresses(storeAddressesChan, false) + } + if len(b.balances)+partialStoreBalances > maxBulkBalances { + storeBalancesChan = make(chan error) + go b.parallelStoreBalances(storeBalancesChan, false) + } + } + b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ + bi: BlockInfo{ + Hash: block.Hash, + Time: block.Time, + Txs: uint32(len(block.Txs)), + Size: uint32(block.Size), + Height: block.Height, + }, + addresses: addresses, + }) + b.bulkAddressesCount += len(addresses) + // open WriteBatch only if going to write + if sa || b.bulkAddressesCount > maxBulkAddresses || storeBlockTxs { + start := time.Now() + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if sa || b.bulkAddressesCount > maxBulkAddresses { + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + } + if storeBlockTxs { + if err := b.d.storeAndCleanupBlockTxs(wb, block); err != nil { + return err + } + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + if bac > b.bulkAddressesCount { + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + } + } + if storeAddressesChan != nil { + if err := <-storeAddressesChan; err != nil { + return err + } + } + if storeBalancesChan != nil { + if err := <-storeBalancesChan; err != nil { + return err + } + } + return nil +} + +// Close flushes the cached data and switches DB from inconsistent state open +// after Close, the BulkConnect cannot be used +func (b *BulkConnect) Close() error { + glog.Info("rocksdb: bulk connect closing") + start := time.Now() + storeAddressesChan := make(chan error) + go b.parallelStoreTxAddresses(storeAddressesChan, true) + storeBalancesChan := make(chan error) + go b.parallelStoreBalances(storeBalancesChan, true) + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + bac := b.bulkAddressesCount + if err := b.storeBulkAddresses(wb); err != nil { + return err + } + if err := b.d.db.Write(b.d.wo, wb); err != nil { + return err + } + glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start)) + if err := <-storeAddressesChan; err != nil { + return err + } + if err := <-storeBalancesChan; err != nil { + return err + } + if err := b.d.SetInconsistentState(false); err != nil { + return err + } + glog.Info("rocksdb: bulk connect closed, db set to open state") + b.d = nil + return nil +} diff --git a/db/dboptions.go b/db/dboptions.go new file mode 100644 index 00000000..f982f8a3 --- /dev/null +++ b/db/dboptions.go @@ -0,0 +1,60 @@ +package db + +import ( + "github.com/tecbot/gorocksdb" +) + +/* + possible additional tuning, using options not accessible by gorocksdb + +// #include "rocksdb/c.h" +import "C" + + cNativeOpts := C.rocksdb_options_create() + opts := &gorocksdb.Options{} + cField := reflect.Indirect(reflect.ValueOf(opts)).FieldByName("c") + cPtr := (**C.rocksdb_options_t)(unsafe.Pointer(cField.UnsafeAddr())) + *cPtr = cNativeOpts + + cNativeBlockOpts := C.rocksdb_block_based_options_create() + blockOpts := &gorocksdb.BlockBasedTableOptions{} + cBlockField := reflect.Indirect(reflect.ValueOf(blockOpts)).FieldByName("c") + cBlockPtr := (**C.rocksdb_block_based_table_options_t)(unsafe.Pointer(cBlockField.UnsafeAddr())) + *cBlockPtr = cNativeBlockOpts + + // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters + blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType) + C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true)) + C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096)) + C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true)) + blockOpts.SetPinL0FilterAndIndexBlocksInCache(true) + +// boolToChar converts a bool value to C.uchar. +func boolToChar(b bool) C.uchar { + if b { + return 1 + } + return 0 +} +*/ + +func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache, maxOpenFiles int) *gorocksdb.Options { + blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() + blockOpts.SetBlockSize(32 << 10) // 32kB + blockOpts.SetBlockCache(c) + if bloomBits > 0 { + blockOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(bloomBits)) + } + opts := gorocksdb.NewDefaultOptions() + opts.SetBlockBasedTableFactory(blockOpts) + opts.SetCreateIfMissing(true) + opts.SetCreateIfMissingColumnFamilies(true) + opts.SetMaxBackgroundCompactions(6) + opts.SetMaxBackgroundFlushes(6) + opts.SetBytesPerSync(8 << 20) // 8MB + opts.SetWriteBufferSize(1 << 27) // 128MB + opts.SetMaxBytesForLevelBase(1 << 27) // 128MB + opts.SetMaxOpenFiles(maxOpenFiles) + opts.SetCompression(gorocksdb.LZ4HCCompression) + return opts +} diff --git a/db/rocksdb.go b/db/rocksdb.go index 30285dab..dd34c6fa 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -6,6 +6,8 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "fmt" + "math/big" "os" "path/filepath" "time" @@ -21,7 +23,8 @@ import ( // when doing huge scan, it is better to close it and reopen from time to time to free the resources const refreshIterator = 5000000 const packedHeightBytes = 4 -const dbVersion = 0 +const dbVersion = 3 +const maxAddrDescLen = 1024 // RepairRocksDB calls RocksDb db repair function func RepairRocksDB(name string) error { @@ -30,67 +33,48 @@ func RepairRocksDB(name string) error { return gorocksdb.RepairDb(name, opts) } +type connectBlockStats struct { + txAddressesHit int + txAddressesMiss int + balancesHit int + balancesMiss int +} + // RocksDB handle type RocksDB struct { - path string - db *gorocksdb.DB - wo *gorocksdb.WriteOptions - ro *gorocksdb.ReadOptions - cfh []*gorocksdb.ColumnFamilyHandle - chainParser bchain.BlockChainParser - is *common.InternalState - metrics *common.Metrics + path string + db *gorocksdb.DB + wo *gorocksdb.WriteOptions + ro *gorocksdb.ReadOptions + cfh []*gorocksdb.ColumnFamilyHandle + chainParser bchain.BlockChainParser + is *common.InternalState + metrics *common.Metrics + cache *gorocksdb.Cache + maxOpenFiles int + cbs connectBlockStats } const ( cfDefault = iota cfHeight cfAddresses - cfUnspentTxs + cfTxAddresses + cfAddressBalance + cfBlockTxs cfTransactions - cfBlockAddresses ) -var cfNames = []string{"default", "height", "addresses", "unspenttxs", "transactions", "blockaddresses"} - -func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { - c := gorocksdb.NewLRUCache(8 << 30) // 8GB - fp := gorocksdb.NewBloomFilter(10) - bbto := gorocksdb.NewDefaultBlockBasedTableOptions() - bbto.SetBlockSize(16 << 10) // 16kB - bbto.SetBlockCache(c) - bbto.SetFilterPolicy(fp) - - opts := gorocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.SetCreateIfMissingColumnFamilies(true) - opts.SetMaxBackgroundCompactions(4) - opts.SetMaxBackgroundFlushes(2) - opts.SetBytesPerSync(1 << 20) // 1MB - opts.SetWriteBufferSize(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) - opts.SetCompression(gorocksdb.NoCompression) - - // opts for outputs are different: - // no bloom filter - from documentation: If most of your queries are executed using iterators, you shouldn't set bloom filter - bbtoOutputs := gorocksdb.NewDefaultBlockBasedTableOptions() - bbtoOutputs.SetBlockSize(16 << 10) // 16kB - bbtoOutputs.SetBlockCache(c) // 8GB - - optsOutputs := gorocksdb.NewDefaultOptions() - optsOutputs.SetBlockBasedTableFactory(bbtoOutputs) - optsOutputs.SetCreateIfMissing(true) - optsOutputs.SetCreateIfMissingColumnFamilies(true) - optsOutputs.SetMaxBackgroundCompactions(4) - optsOutputs.SetMaxBackgroundFlushes(2) - optsOutputs.SetBytesPerSync(1 << 20) // 1MB - optsOutputs.SetWriteBufferSize(1 << 27) // 128MB - optsOutputs.SetMaxOpenFiles(25000) - optsOutputs.SetCompression(gorocksdb.NoCompression) - - fcOptions := []*gorocksdb.Options{opts, opts, optsOutputs, opts, opts, opts} +var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} +func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { + // opts with bloom filter + opts := createAndSetDBOptions(10, c, openFiles) + // opts for addresses without bloom filter + // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter + optsAddresses := createAndSetDBOptions(0, c, openFiles) + // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions + fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) if err != nil { return nil, nil, err @@ -100,13 +84,13 @@ func openDB(path string) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { - glog.Infof("rocksdb: open %s", path) - db, cfh, err := openDB(path) +func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { + glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) + c := gorocksdb.NewLRUCache(cacheSize) + db, cfh, err := openDB(path, c, maxOpenFiles) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() - ro.SetFillCache(false) - return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics}, nil + return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles, connectBlockStats{}}, nil } func (d *RocksDB) closeDB() error { @@ -144,7 +128,7 @@ func (d *RocksDB) Reopen() error { return err } d.db = nil - db, cfh, err := openDB(d.path) + db, cfh, err := openDB(d.path, d.cache, d.maxOpenFiles) if err != nil { return err } @@ -152,19 +136,59 @@ func (d *RocksDB) Reopen() error { return nil } +func (d *RocksDB) GetMemoryStats() string { + type columnStats struct { + name string + indexAndFilter string + memtable string + } + cs := make([]columnStats, len(cfNames)) + for i := 0; i < len(cfNames); i++ { + cs[i].name = cfNames[i] + cs[i].indexAndFilter = d.db.GetPropertyCF("rocksdb.estimate-table-readers-mem", d.cfh[i]) + cs[i].memtable = d.db.GetPropertyCF("rocksdb.cur-size-all-mem-tables", d.cfh[i]) + } + m := struct { + cacheUsage int + pinnedCacheUsage int + indexAndFilter string + memtable string + columns []columnStats + }{ + cacheUsage: d.cache.GetUsage(), + pinnedCacheUsage: d.cache.GetPinnedUsage(), + indexAndFilter: d.db.GetProperty("rocksdb.estimate-table-readers-mem"), + memtable: d.db.GetProperty("rocksdb.cur-size-all-mem-tables"), + columns: cs, + } + return fmt.Sprintf("%+v", m) +} + +// StopIteration is returned by callback function to signal stop of iteration +type StopIteration struct{} + +func (e *StopIteration) Error() string { + return "" +} + // GetTransactions finds all input/output transactions for address // Transaction are passed to callback function. func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { if glog.V(1) { glog.Infof("rocksdb: address get %s %d-%d ", address, lower, higher) } - addrID, err := d.chainParser.GetAddrIDFromAddress(address) + addrDesc, err := d.chainParser.GetAddrDescFromAddress(address) if err != nil { return err } + return d.GetAddrDescTransactions(addrDesc, lower, higher, fn) +} - kstart := packAddressKey(addrID, lower) - kstop := packAddressKey(addrID, higher) +// GetAddrDescTransactions finds all input/output transactions for address descriptor +// Transaction are passed to callback function. +func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { + kstart := packAddressKey(addrDesc, lower) + kstop := packAddressKey(addrDesc, higher) it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) defer it.Close() @@ -185,11 +209,11 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f for _, o := range outpoints { var vout uint32 var isOutput bool - if o.vout < 0 { - vout = uint32(^o.vout) + if o.index < 0 { + vout = uint32(^o.index) isOutput = false } else { - vout = uint32(o.vout) + vout = uint32(o.index) isOutput = true } tx, err := d.chainParser.UnpackTxid(o.btxID) @@ -197,6 +221,9 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f return err } if err := fn(tx, vout, isOutput); err != nil { + if _, ok := err.(*StopIteration); ok { + return nil + } return err } } @@ -234,11 +261,31 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { isUTXO := d.chainParser.IsUTXOChain() - if err := d.writeHeight(wb, block, op); err != nil { + if err := d.writeHeightFromBlock(wb, block, op); err != nil { return err } if isUTXO { - if err := d.writeAddressesUTXO(wb, block, op); err != nil { + if op == opDelete { + // block does not contain mapping tx-> input address, which is necessary to recreate + // unspentTxs; therefore it is not possible to DisconnectBlocks this way + return errors.New("DisconnectBlock is not supported for UTXO chains") + } + addresses := make(map[string][]outpoint) + txAddressesMap := make(map[string]*TxAddresses) + balances := make(map[string]*AddrBalance) + if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { + return err + } + if err := d.storeAddresses(wb, block.Height, addresses); err != nil { + return err + } + if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { + return err + } + if err := d.storeBalances(wb, balances); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { return err } } else { @@ -254,177 +301,139 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { type outpoint struct { btxID []byte - vout int32 + index int32 } -func (d *RocksDB) packBlockAddress(addrID []byte, spentTxs map[string][]outpoint) []byte { - vBuf := make([]byte, vlq.MaxLen32) - vl := packVarint(int32(len(addrID)), vBuf) - blockAddress := append([]byte(nil), vBuf[:vl]...) - blockAddress = append(blockAddress, addrID...) - if spentTxs == nil { - } else { - addrUnspentTxs := spentTxs[string(addrID)] - vl = packVarint(int32(len(addrUnspentTxs)), vBuf) - blockAddress = append(blockAddress, vBuf[:vl]...) - buf := d.packOutpoints(addrUnspentTxs) - blockAddress = append(blockAddress, buf...) - } - return blockAddress +type TxInput struct { + AddrDesc bchain.AddressDescriptor + ValueSat big.Int } -func (d *RocksDB) writeAddressRecords(wb *gorocksdb.WriteBatch, block *bchain.Block, op int, addresses map[string][]outpoint, spentTxs map[string][]outpoint) error { - keep := d.chainParser.KeepBlockAddresses() - blockAddresses := make([]byte, 0) - for addrID, outpoints := range addresses { - baddrID := []byte(addrID) - key := packAddressKey(baddrID, block.Height) - switch op { - case opInsert: - val := d.packOutpoints(outpoints) - wb.PutCF(d.cfh[cfAddresses], key, val) - if keep > 0 { - // collect all addresses be stored in blockaddresses - // they are used in disconnect blocks - blockAddress := d.packBlockAddress(baddrID, spentTxs) - blockAddresses = append(blockAddresses, blockAddress...) - } - case opDelete: - wb.DeleteCF(d.cfh[cfAddresses], key) - } - } - if keep > 0 && op == opInsert { - // write new block address and txs spent in this block - key := packUint(block.Height) - wb.PutCF(d.cfh[cfBlockAddresses], key, blockAddresses) - // cleanup old block address - if block.Height > uint32(keep) { - for rh := block.Height - uint32(keep); rh < block.Height; rh-- { - key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return err - } - if val.Size() == 0 { - break - } - val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockAddresses], key) - } - } - } - return nil +func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { + return p.GetAddressesFromAddrDesc(ti.AddrDesc) } -func (d *RocksDB) addAddrIDToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrID []byte, btxid []byte, vout int32, bh uint32) error { - if len(addrID) > 0 { - if len(addrID) > 1024 { - glog.Infof("rocksdb: block %d, skipping addrID of length %d", bh, len(addrID)) - } else { - strAddrID := string(addrID) - records[strAddrID] = append(records[strAddrID], outpoint{ - btxID: btxid, - vout: vout, - }) - if op == opDelete { - // remove transactions from cache - d.internalDeleteTx(wb, btxid) - } - } - } - return nil +type TxOutput struct { + AddrDesc bchain.AddressDescriptor + Spent bool + ValueSat big.Int } -func (d *RocksDB) getUnspentTx(btxID []byte) ([]byte, error) { - // find it in db, in the column cfUnspentTxs - val, err := d.db.GetCF(d.ro, d.cfh[cfUnspentTxs], btxID) +func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { + return p.GetAddressesFromAddrDesc(to.AddrDesc) +} + +type TxAddresses struct { + Height uint32 + Inputs []TxInput + Outputs []TxOutput +} + +type AddrBalance struct { + Txs uint32 + SentSat big.Int + BalanceSat big.Int +} + +func (ab *AddrBalance) ReceivedSat() *big.Int { + var r big.Int + r.Add(&ab.BalanceSat, &ab.SentSat) + return &r +} + +type blockTxs struct { + btxID []byte + inputs []outpoint +} + +func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc bchain.AddressDescriptor, logText string) { + ad, _, err := d.chainParser.GetAddressesFromAddrDesc(addrDesc) if err != nil { - return nil, err + glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", addrDesc, logText, valueSat.String(), err) + } else { + glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, addrDesc, logText, valueSat.String()) } - defer val.Free() - data := append([]byte(nil), val.Data()...) - return data, nil + valueSat.SetInt64(0) } -func appendPackedAddrID(txAddrs []byte, addrID []byte, n uint32, remaining int) []byte { - // resize the addr buffer if necessary by a new estimate - if cap(txAddrs)-len(txAddrs) < 2*vlq.MaxLen32+len(addrID) { - txAddrs = append(txAddrs, make([]byte, vlq.MaxLen32+len(addrID)+remaining*32)...)[:len(txAddrs)] - } - // addrID is packed as number of bytes of the addrID + bytes of addrID + vout - lv := packVarint(int32(len(addrID)), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - txAddrs = append(txAddrs, addrID...) - lv = packVarint(int32(n), txAddrs[len(txAddrs):len(txAddrs)+vlq.MaxLen32]) - txAddrs = txAddrs[:len(txAddrs)+lv] - return txAddrs +func (d *RocksDB) GetAndResetConnectBlockStats() string { + s := fmt.Sprintf("%+v", d.cbs) + d.cbs = connectBlockStats{} + return s } -func findAndRemoveUnspentAddr(unspentAddrs []byte, vout uint32) ([]byte, []byte) { - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(unspentAddrs); { - l, lv1 := unpackVarint(unspentAddrs[i:]) - // index of vout of address in unspentAddrs - j := i + int(l) + lv1 - if j >= len(unspentAddrs) { - glog.Error("rocksdb: Inconsistent data in unspentAddrs ", hex.EncodeToString(unspentAddrs), ", ", vout) - return nil, unspentAddrs - } - n, lv2 := unpackVarint(unspentAddrs[j:]) - if uint32(n) == vout { - addrID := append([]byte(nil), unspentAddrs[i+lv1:j]...) - unspentAddrs = append(unspentAddrs[:i], unspentAddrs[j+lv2:]...) - return addrID, unspentAddrs - } - i = j + lv2 - } - return nil, unspentAddrs -} - -func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } - addresses := make(map[string][]outpoint) - unspentTxs := make(map[string][]byte) - thisBlockTxs := make(map[string]struct{}) - btxIDs := make([][]byte, len(block.Txs)) - // first process all outputs, build mapping of addresses to outpoints and mappings of unspent txs to addresses - for txi, tx := range block.Txs { +func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { + blockTxIDs := make([][]byte, len(block.Txs)) + blockTxAddresses := make([]*TxAddresses, len(block.Txs)) + // first process all outputs so that inputs can point to txs in this block + for txi := range block.Txs { + tx := &block.Txs[txi] btxID, err := d.chainParser.PackTxid(tx.Txid) if err != nil { return err } - btxIDs[txi] = btxID - // preallocate estimated size of addresses (32 bytes is 1 byte length of addrID, 25 bytes addrID, 1-2 bytes vout and reserve) - txAddrs := make([]byte, 0, len(tx.Vout)*32) + blockTxIDs[txi] = btxID + ta := TxAddresses{Height: block.Height} + ta.Outputs = make([]TxOutput, len(tx.Vout)) + txAddressesMap[string(btxID)] = &ta + blockTxAddresses[txi] = &ta for i, output := range tx.Vout { - addrID, err := d.chainParser.GetAddrIDFromVout(&output) - if err != nil { - // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) - if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + tao := &ta.Outputs[i] + tao.ValueSat = output.ValueSat + addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) + if err != nil || len(addrDesc) == 0 || len(addrDesc) > maxAddrDescLen { + if err != nil { + // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + } + } else { + glog.Infof("rocksdb: height %d, tx %v, vout %v, skipping addrDesc of length %d", block.Height, tx.Txid, i, len(addrDesc)) } continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(output.N), block.Height) - if err != nil { - return err + tao.AddrDesc = addrDesc + strAddrDesc := string(addrDesc) + // check that the address was used already in this block + o, processed := addresses[strAddrDesc] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, btxID) } - txAddrs = appendPackedAddrID(txAddrs, addrID, output.N, len(tx.Vout)-i) + addresses[strAddrDesc] = append(o, outpoint{ + btxID: btxID, + index: int32(i), + }) + ab, e := balances[strAddrDesc] + if !e { + ab, err = d.GetAddrDescBalance(addrDesc) + if err != nil { + return err + } + if ab == nil { + ab = &AddrBalance{} + } + balances[strAddrDesc] = ab + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.Txs++ + } + ab.BalanceSat.Add(&ab.BalanceSat, &output.ValueSat) } - stxID := string(btxID) - unspentTxs[stxID] = txAddrs - thisBlockTxs[stxID] = struct{}{} } - // locate addresses spent by this tx and remove them from unspent addresses - // keep them so that they be stored for DisconnectBlock functionality - spentTxs := make(map[string][]outpoint) - for txi, tx := range block.Txs { - spendingTxid := btxIDs[txi] + // process inputs + for txi := range block.Txs { + tx := &block.Txs[txi] + spendingTxid := blockTxIDs[txi] + ta := blockTxAddresses[txi] + ta.Inputs = make([]TxInput, len(tx.Vin)) + logged := false for i, input := range tx.Vin { + tai := &ta.Inputs[i] btxID, err := d.chainParser.PackTxid(input.Txid) if err != nil { // do not process inputs without input txid @@ -433,50 +442,406 @@ func (d *RocksDB) writeAddressesUTXO(wb *gorocksdb.WriteBatch, block *bchain.Blo } return err } - // find the tx in current block or already processed stxID := string(btxID) - unspentAddrs, exists := unspentTxs[stxID] - if !exists { - // else find it in previous blocks - unspentAddrs, err = d.getUnspentTx(btxID) + ita, e := txAddressesMap[stxID] + if !e { + ita, err = d.getTxAddresses(btxID) if err != nil { return err } - if unspentAddrs == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v missing in unspentTxs", block.Height, tx.Txid, input.Txid, input.Vout, i) + if ita == nil { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v not found in txAddresses", block.Height, tx.Txid, input.Txid) continue } + txAddressesMap[stxID] = ita + d.cbs.txAddressesMiss++ + } else { + d.cbs.txAddressesHit++ } - var addrID []byte - addrID, unspentAddrs = findAndRemoveUnspentAddr(unspentAddrs, input.Vout) - if addrID == nil { - glog.Warningf("rocksdb: height %d, tx %v, input tx %v vin %v %v not found in unspentAddrs", block.Height, tx.Txid, input.Txid, input.Vout, i) + if len(ita.Outputs) <= int(input.Vout) { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) continue } - // record what was spent in this tx - // skip transactions that were created in this block - if _, exists := thisBlockTxs[stxID]; !exists { - saddrID := string(addrID) - rut := spentTxs[saddrID] - rut = append(rut, outpoint{btxID, int32(input.Vout)}) - spentTxs[saddrID] = rut + ot := &ita.Outputs[int(input.Vout)] + if ot.Spent { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, spendingTxid, int32(^i), block.Height) + tai.AddrDesc = ot.AddrDesc + tai.ValueSat = ot.ValueSat + // mark the output as spent in tx + ot.Spent = true + if len(ot.AddrDesc) == 0 { + if !logged { + glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) + logged = true + } + continue + } + strAddrDesc := string(ot.AddrDesc) + // check that the address was used already in this block + o, processed := addresses[strAddrDesc] + if processed { + // check that the address was already used in this tx + processed = processedInTx(o, spendingTxid) + } + addresses[strAddrDesc] = append(o, outpoint{ + btxID: spendingTxid, + index: ^int32(i), + }) + ab, e := balances[strAddrDesc] + if !e { + ab, err = d.GetAddrDescBalance(ot.AddrDesc) + if err != nil { + return err + } + if ab == nil { + ab = &AddrBalance{} + } + balances[strAddrDesc] = ab + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ + } + // add number of trx in balance only once, address can be multiple times in tx + if !processed { + ab.Txs++ + } + ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat) + if ab.BalanceSat.Sign() < 0 { + d.resetValueSatToZero(&ab.BalanceSat, ot.AddrDesc, "balance") + } + ab.SentSat.Add(&ab.SentSat, &ot.ValueSat) + } + } + return nil +} + +func processedInTx(o []outpoint, btxID []byte) bool { + for _, op := range o { + if bytes.Equal(btxID, op.btxID) { + return true + } + } + return false +} + +func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses map[string][]outpoint) error { + for addrDesc, outpoints := range addresses { + ba := bchain.AddressDescriptor(addrDesc) + key := packAddressKey(ba, height) + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + } + return nil +} + +func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*TxAddresses) error { + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) + for txID, ta := range am { + buf = packTxAddresses(ta, buf, varBuf) + wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) + } + return nil +} + +func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBalance) error { + // allocate buffer big enough for number of txs + 2 bigints + buf := make([]byte, vlq.MaxLen32+2*maxPackedBigintBytes) + for addrDesc, ab := range abm { + // balance with 0 transactions is removed from db - happens in disconnect + if ab == nil || ab.Txs <= 0 { + wb.DeleteCF(d.cfh[cfAddressBalance], bchain.AddressDescriptor(addrDesc)) + } else { + l := packVaruint(uint(ab.Txs), buf) + ll := packBigint(&ab.SentSat, buf[l:]) + l += ll + ll = packBigint(&ab.BalanceSat, buf[l:]) + l += ll + wb.PutCF(d.cfh[cfAddressBalance], bchain.AddressDescriptor(addrDesc), buf[:l]) + } + } + return nil +} + +func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, 0, pl*len(block.Txs)) + varBuf := make([]byte, vlq.MaxLen64) + zeroTx := make([]byte, pl) + for i := range block.Txs { + tx := &block.Txs[i] + o := make([]outpoint, len(tx.Vin)) + for v := range tx.Vin { + vin := &tx.Vin[v] + btxID, err := d.chainParser.PackTxid(vin.Txid) + if err != nil { + // do not process inputs without input txid + if err == bchain.ErrTxidMissing { + btxID = zeroTx + } else { + return err + } + } + o[v].btxID = btxID + o[v].index = int32(vin.Vout) + } + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return err + } + buf = append(buf, btxID...) + l := packVaruint(uint(len(o)), varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, d.packOutpoints(o)...) + } + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockTxs], key, buf) + keep := d.chainParser.KeepBlockAddresses() + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh < block.Height; rh-- { + key = packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) if err != nil { return err } - unspentTxs[stxID] = unspentAddrs + if val.Size() == 0 { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) } } - if err := d.writeAddressRecords(wb, block, op, addresses, spentTxs); err != nil { - return err + return nil +} + +func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { + pl := d.chainParser.PackedTxidLen() + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) + if err != nil { + return nil, err } - // save unspent txs from current block - for tx, val := range unspentTxs { - if len(val) == 0 { - wb.DeleteCF(d.cfh[cfUnspentTxs], []byte(tx)) + defer val.Free() + buf := val.Data() + bt := make([]blockTxs, 0) + for i := 0; i < len(buf); { + if len(buf)-i < pl { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + txid := make([]byte, pl) + copy(txid, buf[i:]) + i += pl + o, ol, err := d.unpackNOutpoints(buf[i:]) + if err != nil { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + bt = append(bt, blockTxs{ + btxID: txid, + inputs: o, + }) + i += ol + } + return bt, nil +} + +func (d *RocksDB) GetAddrDescBalance(addrDesc bchain.AddressDescriptor) (*AddrBalance, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrDesc) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 3 is minimum length of addrBalance - 1 byte txs, 1 byte sent, 1 byte balance + if len(buf) < 3 { + return nil, nil + } + txs, l := unpackVaruint(buf) + sentSat, sl := unpackBigint(buf[l:]) + balanceSat, _ := unpackBigint(buf[l+sl:]) + return &AddrBalance{ + Txs: uint32(txs), + SentSat: sentSat, + BalanceSat: balanceSat, + }, nil +} + +// GetAddressBalance returns address balance for an address or nil if address not found +func (d *RocksDB) GetAddressBalance(address string) (*AddrBalance, error) { + addrDesc, err := d.chainParser.GetAddrDescFromAddress(address) + if err != nil { + return nil, err + } + return d.GetAddrDescBalance(addrDesc) +} + +func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // 2 is minimum length of addrBalance - 1 byte height, 1 byte inputs len, 1 byte outputs len + if len(buf) < 3 { + return nil, nil + } + return unpackTxAddresses(buf) +} + +// GetTxAddresses returns TxAddresses for given txid or nil if not found +func (d *RocksDB) GetTxAddresses(txid string) (*TxAddresses, error) { + btxID, err := d.chainParser.PackTxid(txid) + if err != nil { + return nil, err + } + return d.getTxAddresses(btxID) +} + +func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { + buf = buf[:0] + l := packVaruint(uint(ta.Height), varBuf) + buf = append(buf, varBuf[:l]...) + l = packVaruint(uint(len(ta.Inputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.Inputs { + buf = appendTxInput(&ta.Inputs[i], buf, varBuf) + } + l = packVaruint(uint(len(ta.Outputs)), varBuf) + buf = append(buf, varBuf[:l]...) + for i := range ta.Outputs { + buf = appendTxOutput(&ta.Outputs[i], buf, varBuf) + } + return buf +} + +func appendTxInput(txi *TxInput, buf []byte, varBuf []byte) []byte { + la := len(txi.AddrDesc) + l := packVaruint(uint(la), varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txi.AddrDesc...) + l = packBigint(&txi.ValueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { + la := len(txo.AddrDesc) + if txo.Spent { + la = ^la + } + l := packVarint(la, varBuf) + buf = append(buf, varBuf[:l]...) + buf = append(buf, txo.AddrDesc...) + l = packBigint(&txo.ValueSat, varBuf) + buf = append(buf, varBuf[:l]...) + return buf +} + +func unpackTxAddresses(buf []byte) (*TxAddresses, error) { + ta := TxAddresses{} + height, l := unpackVaruint(buf) + ta.Height = uint32(height) + inputs, ll := unpackVaruint(buf[l:]) + l += ll + ta.Inputs = make([]TxInput, inputs) + for i := uint(0); i < inputs; i++ { + l += unpackTxInput(&ta.Inputs[i], buf[l:]) + } + outputs, ll := unpackVaruint(buf[l:]) + l += ll + ta.Outputs = make([]TxOutput, outputs) + for i := uint(0); i < outputs; i++ { + l += unpackTxOutput(&ta.Outputs[i], buf[l:]) + } + return &ta, nil +} + +func unpackTxInput(ti *TxInput, buf []byte) int { + al, l := unpackVaruint(buf) + ti.AddrDesc = make([]byte, al) + copy(ti.AddrDesc, buf[l:l+int(al)]) + al += uint(l) + ti.ValueSat, l = unpackBigint(buf[al:]) + return l + int(al) +} + +func unpackTxOutput(to *TxOutput, buf []byte) int { + al, l := unpackVarint(buf) + if al < 0 { + to.Spent = true + al = ^al + } + to.AddrDesc = make([]byte, al) + copy(to.AddrDesc, buf[l:l+al]) + al += l + to.ValueSat, l = unpackBigint(buf[al:]) + return l + al +} + +func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { + buf := make([]byte, 0) + bvout := make([]byte, vlq.MaxLen32) + for _, o := range outpoints { + l := packVarint32(o.index, bvout) + buf = append(buf, []byte(o.btxID)...) + buf = append(buf, bvout[:l]...) + } + return buf +} + +func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + outpoints := make([]outpoint, 0) + for i := 0; i < len(buf); { + btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) + i += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[i:]) + i += voutLen + outpoints = append(outpoints, outpoint{ + btxID: btxID, + index: vout, + }) + } + return outpoints, nil +} + +func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { + txidUnpackedLen := d.chainParser.PackedTxidLen() + n, p := unpackVaruint(buf) + outpoints := make([]outpoint, n) + for i := uint(0); i < n; i++ { + if p+txidUnpackedLen >= len(buf) { + return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") + } + btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) + p += txidUnpackedLen + vout, voutLen := unpackVarint32(buf[p:]) + p += voutLen + outpoints[i] = outpoint{ + btxID: btxID, + index: vout, + } + } + return outpoints, p, nil +} + +func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc bchain.AddressDescriptor, btxid []byte, vout int32, bh uint32) error { + if len(addrDesc) > 0 { + if len(addrDesc) > maxAddrDescLen { + glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc)) } else { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(tx), val) + strAddrDesc := string(addrDesc) + records[strAddrDesc] = append(records[strAddrDesc], outpoint{ + btxID: btxid, + index: vout, + }) + if op == opDelete { + // remove transactions from cache + d.internalDeleteTx(wb, btxid) + } } } return nil @@ -490,15 +855,15 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. return err } for _, output := range tx.Vout { - addrID, err := d.chainParser.GetAddrIDFromVout(&output) + addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) if err != nil { // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrID: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) } continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(output.N), block.Height) + err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(output.N), block.Height) if err != nil { return err } @@ -506,125 +871,92 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // store inputs in format txid ^index for _, input := range tx.Vin { for i, a := range input.Addresses { - addrID, err := d.chainParser.GetAddrIDFromAddress(a) + addrDesc, err := d.chainParser.GetAddrDescFromAddress(a) if err != nil { - glog.Warningf("rocksdb: addrID: %v - %d %s", err, block.Height, addrID) + glog.Warningf("rocksdb: addrDesc: %v - %d %s", err, block.Height, addrDesc) continue } - err = d.addAddrIDToRecords(op, wb, addresses, addrID, btxID, int32(^i), block.Height) + err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(^i), block.Height) if err != nil { return err } } } } - return d.writeAddressRecords(wb, block, op, addresses, nil) -} - -func (d *RocksDB) unpackBlockAddresses(buf []byte) ([][]byte, [][]outpoint, error) { - addresses := make([][]byte, 0) - outpointsArray := make([][]outpoint, 0) - // the addresses are packed as lenaddrID addrID vout, where lenaddrID and vout are varints - for i := 0; i < len(buf); { - l, lv := unpackVarint(buf[i:]) - j := i + int(l) + lv - if j > len(buf) { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addrID := append([]byte(nil), buf[i+lv:j]...) - outpoints, ol, err := d.unpackNOutpoints(buf[j:]) - if err != nil { - glog.Error("rocksdb: Inconsistent data in blockAddresses ", hex.EncodeToString(buf)) - return nil, nil, errors.New("Inconsistent data in blockAddresses") - } - addresses = append(addresses, addrID) - outpointsArray = append(outpointsArray, outpoints) - i = j + ol - } - return addresses, outpointsArray, nil -} - -func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { - buf := make([]byte, 0) - bvout := make([]byte, vlq.MaxLen32) - for _, o := range outpoints { - l := packVarint(o.vout, bvout) - buf = append(buf, []byte(o.btxID)...) - buf = append(buf, bvout[:l]...) - } - return buf -} - -func (d *RocksDB) unpackOutpoints(buf []byte) ([]outpoint, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - outpoints := make([]outpoint, 0) - for i := 0; i < len(buf); { - btxID := append([]byte(nil), buf[i:i+txidUnpackedLen]...) - i += txidUnpackedLen - vout, voutLen := unpackVarint(buf[i:]) - i += voutLen - outpoints = append(outpoints, outpoint{ - btxID: btxID, - vout: vout, - }) - } - return outpoints, nil -} - -func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - n, p := unpackVarint(buf) - outpoints := make([]outpoint, n) - for i := int32(0); i < n; i++ { - if p+txidUnpackedLen >= len(buf) { - return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") - } - btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) - p += txidUnpackedLen - vout, voutLen := unpackVarint(buf[p:]) - p += voutLen - outpoints[i] = outpoint{ - btxID: btxID, - vout: vout, + for addrDesc, outpoints := range addresses { + key := packAddressKey(bchain.AddressDescriptor(addrDesc), block.Height) + switch op { + case opInsert: + val := d.packOutpoints(outpoints) + wb.PutCF(d.cfh[cfAddresses], key, val) + case opDelete: + wb.DeleteCF(d.cfh[cfAddresses], key) } } - return outpoints, p, nil -} - -func (d *RocksDB) packOutpoint(txid string, vout int32) ([]byte, error) { - btxid, err := d.chainParser.PackTxid(txid) - if err != nil { - return nil, err - } - bv := make([]byte, vlq.MaxLen32) - l := packVarint(vout, bv) - buf := make([]byte, 0, l+len(btxid)) - buf = append(buf, btxid...) - buf = append(buf, bv[:l]...) - return buf, nil -} - -func (d *RocksDB) unpackOutpoint(buf []byte) (string, int32, int) { - txidUnpackedLen := d.chainParser.PackedTxidLen() - txid, _ := d.chainParser.UnpackTxid(buf[:txidUnpackedLen]) - vout, o := unpackVarint(buf[txidUnpackedLen:]) - return txid, vout, txidUnpackedLen + o + return nil } // Block index +// BlockInfo holds information about blocks kept in column height +type BlockInfo struct { + Hash string + Time int64 + Txs uint32 + Size uint32 + Height uint32 // Height is not packed! +} + +func (d *RocksDB) packBlockInfo(block *BlockInfo) ([]byte, error) { + packed := make([]byte, 0, 64) + varBuf := make([]byte, vlq.MaxLen64) + b, err := d.chainParser.PackBlockHash(block.Hash) + if err != nil { + return nil, err + } + packed = append(packed, b...) + packed = append(packed, packUint(uint32(block.Time))...) + l := packVaruint(uint(block.Txs), varBuf) + packed = append(packed, varBuf[:l]...) + l = packVaruint(uint(block.Size), varBuf) + packed = append(packed, varBuf[:l]...) + return packed, nil +} + +func (d *RocksDB) unpackBlockInfo(buf []byte) (*BlockInfo, error) { + pl := d.chainParser.PackedTxidLen() + // minimum length is PackedTxidLen+4 bytes time + 1 byte txs + 1 byte size + if len(buf) < pl+4+2 { + return nil, nil + } + txid, err := d.chainParser.UnpackBlockHash(buf[:pl]) + if err != nil { + return nil, err + } + t := unpackUint(buf[pl:]) + txs, l := unpackVaruint(buf[pl+4:]) + size, _ := unpackVaruint(buf[pl+4+l:]) + return &BlockInfo{ + Hash: txid, + Time: int64(t), + Txs: uint32(txs), + Size: uint32(size), + }, nil +} + // GetBestBlock returns the block hash of the block with highest height in the db func (d *RocksDB) GetBestBlock() (uint32, string, error) { it := d.db.NewIteratorCF(d.ro, d.cfh[cfHeight]) defer it.Close() if it.SeekToLast(); it.Valid() { bestHeight := unpackUint(it.Key().Data()) - val, err := d.chainParser.UnpackBlockHash(it.Value().Data()) - if glog.V(1) { - glog.Infof("rocksdb: bestblock %d %s", bestHeight, val) + info, err := d.unpackBlockInfo(it.Value().Data()) + if info != nil { + if glog.V(1) { + glog.Infof("rocksdb: bestblock %d %+v", bestHeight, info) + } + return bestHeight, info.Hash, err } - return bestHeight, val, err } return 0, "", nil } @@ -637,42 +969,57 @@ func (d *RocksDB) GetBlockHash(height uint32) (string, error) { return "", err } defer val.Free() - return d.chainParser.UnpackBlockHash(val.Data()) + info, err := d.unpackBlockInfo(val.Data()) + if info == nil { + return "", err + } + return info.Hash, nil } -func (d *RocksDB) writeHeight( - wb *gorocksdb.WriteBatch, - block *bchain.Block, - op int, -) error { - key := packUint(block.Height) +// GetBlockInfo returns block info stored in db +func (d *RocksDB) GetBlockInfo(height uint32) (*BlockInfo, error) { + key := packUint(height) + val, err := d.db.GetCF(d.ro, d.cfh[cfHeight], key) + if err != nil { + return nil, err + } + defer val.Free() + bi, err := d.unpackBlockInfo(val.Data()) + if err != nil { + return nil, err + } + bi.Height = height + return bi, err +} +func (d *RocksDB) writeHeightFromBlock(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { + return d.writeHeight(wb, block.Height, &BlockInfo{ + Hash: block.Hash, + Time: block.Time, + Txs: uint32(len(block.Txs)), + Size: uint32(block.Size), + Height: block.Height, + }, op) +} + +func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, height uint32, bi *BlockInfo, op int) error { + key := packUint(height) switch op { case opInsert: - val, err := d.chainParser.PackBlockHash(block.Hash) + val, err := d.packBlockInfo(bi) if err != nil { return err } wb.PutCF(d.cfh[cfHeight], key, val) + d.is.UpdateBestHeight(height) case opDelete: wb.DeleteCF(d.cfh[cfHeight], key) + d.is.UpdateBestHeight(height - 1) } - return nil } -func (d *RocksDB) getBlockAddresses(key []byte) ([][]byte, [][]outpoint, error) { - b, err := d.db.GetCF(d.ro, d.cfh[cfBlockAddresses], key) - if err != nil { - return nil, nil, err - } - defer b.Free() - // block is missing in DB - if b.Data() == nil { - return nil, nil, errors.New("Block addresses missing") - } - return d.unpackBlockAddresses(b.Data()) -} +// Disconnect blocks func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]byte, error) { glog.Infof("db: doing full scan of addresses column") @@ -719,93 +1066,176 @@ func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]b return addrKeys, addrValues, nil } -// DisconnectBlockRange removes all data belonging to blocks in range lower-higher -// it finds the data in blockaddresses column if available, -// otherwise by doing quite slow full scan of addresses column -func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys := [][]byte{} - addrOutpoints := [][]byte{} - addrUnspentOutpoints := [][]outpoint{} - keep := d.chainParser.KeepBlockAddresses() - var err error - if keep > 0 { - for height := lower; height <= higher; height++ { - addresses, unspentOutpoints, err := d.getBlockAddresses(packUint(height)) +func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *TxAddresses, + txAddressesToUpdate map[string]*TxAddresses, balances map[string]*AddrBalance) error { + addresses := make(map[string]struct{}) + getAddressBalance := func(addrDesc bchain.AddressDescriptor) (*AddrBalance, error) { + var err error + s := string(addrDesc) + b, fb := balances[s] + if !fb { + b, err = d.GetAddrDescBalance(addrDesc) + if err != nil { + return nil, err + } + balances[s] = b + } + return b, nil + } + for i, t := range txa.Inputs { + if len(t.AddrDesc) > 0 { + s := string(t.AddrDesc) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.AddrDesc) if err != nil { - glog.Error(err) return err } - for i, addrID := range addresses { - addrKey := packAddressKey(addrID, height) - val, err := d.db.GetCF(d.ro, d.cfh[cfAddresses], addrKey) + if b != nil { + // subtract number of txs only once + if !exist { + b.Txs-- + } + b.SentSat.Sub(&b.SentSat, &t.ValueSat) + if b.SentSat.Sign() < 0 { + d.resetValueSatToZero(&b.SentSat, t.AddrDesc, "sent amount") + } + b.BalanceSat.Add(&b.BalanceSat, &t.ValueSat) + } else { + ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) + glog.Warningf("Balance for address %s (%s) not found", ad, t.AddrDesc) + } + s = string(inputs[i].btxID) + sa, exist := txAddressesToUpdate[s] + if !exist { + sa, err = d.getTxAddresses(inputs[i].btxID) if err != nil { - glog.Error(err) return err } - addrKeys = append(addrKeys, addrKey) - av := append([]byte(nil), val.Data()...) - val.Free() - addrOutpoints = append(addrOutpoints, av) - addrUnspentOutpoints = append(addrUnspentOutpoints, unspentOutpoints[i]) + txAddressesToUpdate[s] = sa + } + sa.Outputs[inputs[i].index].Spent = false + } + } + for _, t := range txa.Outputs { + if len(t.AddrDesc) > 0 { + s := string(t.AddrDesc) + _, exist := addresses[s] + if !exist { + addresses[s] = struct{}{} + } + b, err := getAddressBalance(t.AddrDesc) + if err != nil { + return err + } + if b != nil { + // subtract number of txs only once + if !exist { + b.Txs-- + } + b.BalanceSat.Sub(&b.BalanceSat, &t.ValueSat) + if b.BalanceSat.Sign() < 0 { + d.resetValueSatToZero(&b.BalanceSat, t.AddrDesc, "balance") + } + } else { + ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) + glog.Warningf("Balance for address %s (%s) not found", ad, t.AddrDesc) } } - } else { - addrKeys, addrOutpoints, err = d.allAddressesScan(lower, higher) + } + for a := range addresses { + key := packAddressKey([]byte(a), height) + wb.DeleteCF(d.cfh[cfAddresses], key) + } + return nil +} + +// DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher +// if they are in the range kept in the cfBlockTxids column +func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + blocks := make([][]blockTxs, higher-lower+1) + for height := lower; height <= higher; height++ { + blockTxs, err := d.getBlockTxs(height) if err != nil { return err } + if len(blockTxs) == 0 { + return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) + } + blocks[height-lower] = blockTxs } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + txAddressesToUpdate := make(map[string]*TxAddresses) + txsToDelete := make(map[string]struct{}) + balances := make(map[string]*AddrBalance) + for height := higher; height >= lower; height-- { + blockTxs := blocks[height-lower] + glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") + // go backwards to avoid interim negative balance + // when connecting block, amount is first in tx on the output side, then in another tx on the input side + // when disconnecting, it must be done backwards + for i := len(blockTxs) - 1; i >= 0; i-- { + txid := blockTxs[i].btxID + s := string(txid) + txsToDelete[s] = struct{}{} + txa, err := d.getTxAddresses(txid) + if err != nil { + return err + } + if txa == nil { + ut, _ := d.chainParser.UnpackTxid(txid) + glog.Warning("TxAddress for txid ", ut, " not found") + continue + } + if err := d.disconnectTxAddresses(wb, height, s, blockTxs[i].inputs, txa, txAddressesToUpdate, balances); err != nil { + return err + } + } + key := packUint(height) + wb.DeleteCF(d.cfh[cfBlockTxs], key) + wb.DeleteCF(d.cfh[cfHeight], key) + } + d.storeTxAddresses(wb, txAddressesToUpdate) + d.storeBalances(wb, balances) + for s := range txsToDelete { + b := []byte(s) + wb.DeleteCF(d.cfh[cfTransactions], b) + wb.DeleteCF(d.cfh[cfTxAddresses], b) + } + err := d.db.Write(d.wo, wb) + if err == nil { + glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) + } + return err +} +// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks +// it is very slow operation +func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { + glog.Infof("db: disconnecting blocks %d-%d", lower, higher) + addrKeys, _, err := d.allAddressesScan(lower, higher) + if err != nil { + return err + } glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) wb := gorocksdb.NewWriteBatch() defer wb.Destroy() - unspentTxs := make(map[string][]byte) - for addrIndex, addrKey := range addrKeys { + for _, addrKey := range addrKeys { if glog.V(2) { glog.Info("address ", hex.EncodeToString(addrKey)) } // delete address:height from the index wb.DeleteCF(d.cfh[cfAddresses], addrKey) - addrID, _, err := unpackAddressKey(addrKey) - if err != nil { - return err - } - // recreate unspentTxs, which were spent by this block (that is being disconnected) - for _, o := range addrUnspentOutpoints[addrIndex] { - stxID := string(o.btxID) - txAddrs, exists := unspentTxs[stxID] - if !exists { - txAddrs, err = d.getUnspentTx(o.btxID) - if err != nil { - return err - } - } - txAddrs = appendPackedAddrID(txAddrs, addrID, uint32(o.vout), 1) - unspentTxs[stxID] = txAddrs - } - // delete unspentTxs from this block - outpoints, err := d.unpackOutpoints(addrOutpoints[addrIndex]) - if err != nil { - return err - } - for _, o := range outpoints { - wb.DeleteCF(d.cfh[cfUnspentTxs], o.btxID) - d.internalDeleteTx(wb, o.btxID) - } - } - for key, val := range unspentTxs { - wb.PutCF(d.cfh[cfUnspentTxs], []byte(key), val) } for height := lower; height <= higher; height++ { if glog.V(2) { glog.Info("height ", height) } - key := packUint(height) - if keep > 0 { - wb.DeleteCF(d.cfh[cfBlockAddresses], key) - } - wb.DeleteCF(d.cfh[cfHeight], key) + wb.DeleteCF(d.cfh[cfHeight], packUint(height)) } err = d.db.Write(d.wo, wb) if err == nil { @@ -817,8 +1247,10 @@ func (d *RocksDB) DisconnectBlockRange(lower uint32, higher uint32) error { func dirSize(path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { - if !info.IsDir() { - size += info.Size() + if err == nil { + if !info.IsDir() { + size += info.Size() + } } return err }) @@ -945,9 +1377,27 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro } } is.DbColumns = nc + // after load, reset the synchronization data + is.IsSynchronized = false + is.IsMempoolSynchronized = false + var t time.Time + is.LastMempoolSync = t + is.SyncMode = false return is, nil } +func (d *RocksDB) SetInconsistentState(inconsistent bool) error { + if d.is == nil { + return errors.New("Internal state not created") + } + if inconsistent { + d.is.DbState = common.DbStateInconsistent + } else { + d.is.DbState = common.DbStateOpen + } + return d.storeState(d.is) +} + // SetInternalState sets the InternalState to be used by db to collect internal state func (d *RocksDB) SetInternalState(is *common.InternalState) { d.is = is @@ -955,11 +1405,17 @@ func (d *RocksDB) SetInternalState(is *common.InternalState) { // StoreInternalState stores the internal state to db func (d *RocksDB) StoreInternalState(is *common.InternalState) error { - for c := 0; c < len(cfNames); c++ { - rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) - d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) - d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + if d.metrics != nil { + for c := 0; c < len(cfNames); c++ { + rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) + d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) + d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) + } } + return d.storeState(is) +} + +func (d *RocksDB) storeState(is *common.InternalState) error { buf, err := is.Pack() if err != nil { return err @@ -970,9 +1426,12 @@ func (d *RocksDB) StoreInternalState(is *common.InternalState) error { func (d *RocksDB) computeColumnSize(col int, stopCompute chan os.Signal) (int64, int64, int64, error) { var rows, keysSum, valuesSum int64 var seekKey []byte + // do not use cache + ro := gorocksdb.NewDefaultReadOptions() + ro.SetFillCache(false) for { var key []byte - it := d.db.NewIteratorCF(d.ro, d.cfh[col]) + it := d.db.NewIteratorCF(ro, d.cfh[col]) if rows == 0 { it.SeekToFirst() } else { @@ -1021,10 +1480,10 @@ func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) er // Helpers -func packAddressKey(addrID []byte, height uint32) []byte { +func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte { bheight := packUint(height) - buf := make([]byte, 0, len(addrID)+len(bheight)) - buf = append(buf, addrID...) + buf := make([]byte, 0, len(addrDesc)+len(bheight)) + buf = append(buf, addrDesc...) buf = append(buf, bheight...) return buf } @@ -1047,11 +1506,89 @@ func unpackUint(buf []byte) uint32 { return binary.BigEndian.Uint32(buf) } -func packVarint(i int32, buf []byte) int { +func packVarint32(i int32, buf []byte) int { return vlq.PutInt(buf, int64(i)) } -func unpackVarint(buf []byte) (int32, int) { +func packVarint(i int, buf []byte) int { + return vlq.PutInt(buf, int64(i)) +} + +func packVaruint(i uint, buf []byte) int { + return vlq.PutUint(buf, uint64(i)) +} + +func unpackVarint32(buf []byte) (int32, int) { i, ofs := vlq.Int(buf) return int32(i), ofs } + +func unpackVarint(buf []byte) (int, int) { + i, ofs := vlq.Int(buf) + return int(i), ofs +} + +func unpackVaruint(buf []byte) (uint, int) { + i, ofs := vlq.Uint(buf) + return uint(i), ofs +} + +const ( + // number of bits in a big.Word + wordBits = 32 << (uint64(^big.Word(0)) >> 63) + // number of bytes in a big.Word + wordBytes = wordBits / 8 + // max packed bigint words + maxPackedBigintWords = (256 - wordBytes) / wordBytes + maxPackedBigintBytes = 249 +) + +// big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int +// number of written bytes is returned +// limitation: bigints longer than 248 bytes are truncated to 248 bytes +// caution: buffer must be big enough to hold the packed big int, buffer 249 bytes big is always safe +func packBigint(bi *big.Int, buf []byte) int { + w := bi.Bits() + lw := len(w) + // zero returns only one byte - zero length + if lw == 0 { + buf[0] = 0 + return 1 + } + // pack the most significant word in a special way - skip leading zeros + w0 := w[lw-1] + fb := 8 + mask := big.Word(0xff) << (wordBits - 8) + for w0&mask == 0 { + fb-- + mask >>= 8 + } + for i := fb; i > 0; i-- { + buf[i] = byte(w0) + w0 >>= 8 + } + // if the big int is too big (> 2^1984), the number of bytes would not fit to 1 byte + // in this case, truncate the number, it is not expected to work with this big numbers as amounts + s := 0 + if lw > maxPackedBigintWords { + s = lw - maxPackedBigintWords + } + // pack the rest of the words in reverse order + for j := lw - 2; j >= s; j-- { + d := w[j] + for i := fb + wordBytes; i > fb; i-- { + buf[i] = byte(d) + d >>= 8 + } + fb += wordBytes + } + buf[0] = byte(fb) + return fb + 1 +} + +func unpackBigint(buf []byte) (big.Int, int) { + var r big.Int + l := int(buf[0]) + 1 + r.SetBytes(buf[1:l]) + return r, l +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 52be1c5d..8e114597 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -5,16 +5,19 @@ package db import ( "blockbook/bchain" "blockbook/bchain/coins/btc" + "blockbook/common" + "encoding/binary" "encoding/hex" "fmt" "io/ioutil" + "math/big" "os" "reflect" "sort" - "strconv" "strings" "testing" + vlq "github.com/bsm/go-vlq" "github.com/juju/errors" ) @@ -22,12 +25,18 @@ import ( // for number n, the packing is: 2*n if n>=0 else 2*(-n)-1 // takes only 1 byte if abs(n)<127 +func bitcoinTestnetParser() *btc.BitcoinParser { + return btc.NewBitcoinParser( + btc.GetChainParams("test"), + &btc.Configuration{BlockAddressesToKeep: 1}) +} + func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { tmp, err := ioutil.TempDir("", "testdb") if err != nil { t.Fatal(err) } - d, err := NewRocksDB(tmp, p, nil) + d, err := NewRocksDB(tmp, 100000, -1, p, nil) if err != nil { t.Fatal(err) } @@ -47,16 +56,47 @@ func closeAndDestroyRocksDB(t *testing.T, d *RocksDB) { } func addressToPubKeyHex(addr string, t *testing.T, d *RocksDB) string { - b, err := d.chainParser.AddressToOutputScript(addr) + if addr == "" { + return "" + } + b, err := d.chainParser.GetAddrDescFromAddress(addr) if err != nil { t.Fatal(err) } return hex.EncodeToString(b) } +func inputAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return hex.EncodeToString([]byte{byte(len(h) / 2)}) + h +} + func addressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { h := addressToPubKeyHex(addr, t, d) - return strconv.FormatInt(int64(len(h)), 16) + h + return hex.EncodeToString([]byte{byte(len(h))}) + h +} + +func spentAddressToPubKeyHexWithLength(addr string, t *testing.T, d *RocksDB) string { + h := addressToPubKeyHex(addr, t, d) + return hex.EncodeToString([]byte{byte(len(h) + 1)}) + h +} + +func bigintToHex(i *big.Int) string { + b := make([]byte, maxPackedBigintBytes) + l := packBigint(i, b) + return hex.EncodeToString(b[:l]) +} + +func varuintToHex(i uint) string { + b := make([]byte, vlq.MaxLen64) + l := vlq.PutUint(b, uint64(i)) + return hex.EncodeToString(b[:l]) +} + +func uintToHex(i uint32) string { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, i) + return hex.EncodeToString(buf) } // keyPair is used to compare given key value in DB with expected @@ -104,7 +144,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { valOK = kp[i].CompareFunc(val) } if !valOK { - return errors.Errorf("Incorrect value %v found in column %v row %v, expecting %v", val, cfNames[col], i, kp[i].Value) + return errors.Errorf("Incorrect value %v found in column %v row %v key %v, expecting %v", val, cfNames[col], i, key, kp[i].Value) } i++ } @@ -114,52 +154,94 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } +const ( + txidB1T1 = "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + txidB1T2 = "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + txidB2T1 = "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + txidB2T2 = "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + txidB2T3 = "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + txidB2T4 = "fdd824a780cbb718eeb766eb05d83fdefc793a27082cd5e67f856d69798cf7db" + + addr1 = "mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti" // 76a914010d39800f86122416e28f485029acf77507169288ac + addr2 = "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz" // 76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac + addr3 = "mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw" // 76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac + addr4 = "2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS" // a9144a21db08fb6882cb152e1ff06780a430740f770487 + addr5 = "2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1" // a914e921fc4912a315078f370d959f2c4f7b6d2a683c87 + addr6 = "mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX" // 76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac + addr7 = "mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL" // 76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac + addr8 = "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC" // 76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac + addr9 = "mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP" // 76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac + addrA = "mzVznVsCHkVHX9UN8WPFASWUUHtxnNn4Jj" // 76a914d03c0d863d189b23b061a95ad32940b65837609f88ac +) + +var ( + satZero = big.NewInt(0) + satB1T1A1 = big.NewInt(100000000) + satB1T1A2 = big.NewInt(12345) + satB1T2A3 = big.NewInt(1234567890123) + satB1T2A4 = big.NewInt(1) + satB1T2A5 = big.NewInt(9876) + satB2T1A6 = big.NewInt(317283951061) + satB2T1A7 = big.NewInt(917283951061) + satB2T2A8 = big.NewInt(118641975500) + satB2T2A9 = big.NewInt(198641975500) + satB2T3A5 = big.NewInt(9000) + satB2T4AA = big.NewInt(1360030331) +) + func getTestUTXOBlock1(t *testing.T, d *RocksDB) *bchain.Block { return &bchain.Block{ BlockHeader: bchain.BlockHeader{ Height: 225493, Hash: "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", + Size: 1234567, + Time: 1534858021, }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d), + Hex: addressToPubKeyHex(addr1, t, d), }, + ValueSat: *satB1T1A1, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d), + Hex: addressToPubKeyHex(addr2, t, d), }, + ValueSat: *satB1T1A2, }, }, Blocktime: 22549300000, Time: 22549300000, }, bchain.Tx{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: []bchain.Vout{ bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d), + Hex: addressToPubKeyHex(addr3, t, d), }, + ValueSat: *satB1T2A3, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d), + Hex: addressToPubKeyHex(addr4, t, d), }, + ValueSat: *satB1T2A4, }, bchain.Vout{ N: 2, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB1T2A5, }, }, Blocktime: 22549300001, @@ -174,17 +256,21 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { BlockHeader: bchain.BlockHeader{ Height: 225494, Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Size: 2345678, + Time: 1534859123, }, Txs: []bchain.Tx{ bchain.Tx{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vin: []bchain.Vin{ + // addr3 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 0, }, + // addr2 bchain.Vin{ - Txid: "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", + Txid: txidB1T1, Vout: 1, }, }, @@ -192,30 +278,32 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d), + Hex: addressToPubKeyHex(addr6, t, d), }, + ValueSat: *satB2T1A6, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d), + Hex: addressToPubKeyHex(addr7, t, d), }, + ValueSat: *satB2T1A7, }, }, Blocktime: 22549400000, Time: 22549400000, }, bchain.Tx{ - Txid: "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", + Txid: txidB2T2, Vin: []bchain.Vin{ - // spending an output in the same block + // spending an output in the same block - addr6 bchain.Vin{ - Txid: "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", + Txid: txidB2T1, Vout: 0, }, - // spending an output in the previous block + // spending an output in the previous block - addr4 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 1, }, }, @@ -223,14 +311,16 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d), + Hex: addressToPubKeyHex(addr8, t, d), }, + ValueSat: *satB2T2A8, }, bchain.Vout{ N: 1, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d), + Hex: addressToPubKeyHex(addr9, t, d), }, + ValueSat: *satB2T2A9, }, }, Blocktime: 22549400001, @@ -238,10 +328,11 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { }, // transaction from the same address in the previous block bchain.Tx{ - Txid: "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", + Txid: txidB2T3, Vin: []bchain.Vin{ + // addr5 bchain.Vin{ - Txid: "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", + Txid: txidB1T2, Vout: 2, }, }, @@ -249,20 +340,50 @@ func getTestUTXOBlock2(t *testing.T, d *RocksDB) *bchain.Block { bchain.Vout{ N: 0, ScriptPubKey: bchain.ScriptPubKey{ - Hex: addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d), + Hex: addressToPubKeyHex(addr5, t, d), }, + ValueSat: *satB2T3A5, }, }, Blocktime: 22549400002, Time: 22549400002, }, + // mining transaction + bchain.Tx{ + Txid: txidB2T4, + Vin: []bchain.Vin{ + bchain.Vin{ + Coinbase: "03bf1e1504aede765b726567696f6e312f50726f6a65637420425443506f6f6c2f01000001bf7e000000000000", + }, + }, + Vout: []bchain.Vout{ + bchain.Vout{ + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: addressToPubKeyHex(addrA, t, d), + }, + ValueSat: *satB2T4AA, + }, + bchain.Vout{ + N: 1, + ScriptPubKey: bchain.ScriptPubKey{}, + ValueSat: *satZero, + }, + }, + Blocktime: 22549400003, + Time: 22549400003, + }, }, } } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { +func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, + keyPair{ + "000370d5", + "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997" + uintToHex(1534858021) + varuintToHex(2) + varuintToHex(1234567), + nil, + }, }); err != nil { { t.Fatal(err) @@ -270,62 +391,67 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { } // the vout is encoded as signed varint, i.e. value * 2 for non negative values if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02", - }) - }, + txidB1T1, + varuintToHex(225493) + + "00" + + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + addressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), + nil, }, keyPair{ - "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "04", - }) - }, + txidB1T2, + varuintToHex(225493) + + "00" + + "03" + + addressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + addressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), + nil, }, }); err != nil { { t.Fatal(err) } } - // after disconnect there are no blockaddresses for the previous block - var blockAddressesKp []keyPair - if noBlockAddresses { - blockAddressesKp = []keyPair{} + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A2), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A3), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A4), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T2A5), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + var blockTxsKp []keyPair + if afterDisconnect { + blockTxsKp = []keyPair{} } else { - // the values in cfBlockAddresses are in random order, must use CompareFunc - blockAddressesKp = []keyPair{ - keyPair{"000370d5", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "00", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "00", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", - }) - }, + blockTxsKp = []keyPair{ + keyPair{ + "000370d5", + txidB1T1 + "00" + txidB1T2 + "00", + nil, }, } } - if err := checkColumn(d, cfBlockAddresses, blockAddressesKp); err != nil { + + if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil { { t.Fatal(err) } @@ -334,55 +460,99 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, noBlockAddresses bool) { func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ - keyPair{"000370d5", "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997", nil}, - keyPair{"000370d6", "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", nil}, + keyPair{ + "000370d5", + "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997" + uintToHex(1534858021) + varuintToHex(2) + varuintToHex(1234567), + nil, + }, + keyPair{ + "000370d6", + "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6" + uintToHex(1534859123) + varuintToHex(4) + varuintToHex(2345678), + nil, + }, }); err != nil { { t.Fatal(err) } } if err := checkColumn(d, cfAddresses, []keyPair{ - keyPair{addressToPubKeyHex("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "00", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d5", "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d5", "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", nil}, - keyPair{addressToPubKeyHex("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "00" + "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "01", nil}, - keyPair{addressToPubKeyHex("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "02", nil}, - keyPair{addressToPubKeyHex("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "00", nil}, - keyPair{addressToPubKeyHex("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "02", nil}, - keyPair{addressToPubKeyHex("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "01", nil}, - keyPair{addressToPubKeyHex("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "000370d6", "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25" + "03", nil}, - keyPair{addressToPubKeyHex("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "000370d6", "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "00" + "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07" + "01", nil}, - keyPair{addressToPubKeyHex("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "000370d6", "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71" + "03", nil}, + keyPair{addressToPubKeyHex(addr1, t, d) + "000370d5", txidB1T1 + "00", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d5", txidB1T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d5", txidB1T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d5", txidB1T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d5", txidB1T2 + "04", nil}, + keyPair{addressToPubKeyHex(addr6, t, d) + "000370d6", txidB2T1 + "00" + txidB2T2 + "01", nil}, + keyPair{addressToPubKeyHex(addr7, t, d) + "000370d6", txidB2T1 + "02", nil}, + keyPair{addressToPubKeyHex(addr8, t, d) + "000370d6", txidB2T2 + "00", nil}, + keyPair{addressToPubKeyHex(addr9, t, d) + "000370d6", txidB2T2 + "02", nil}, + keyPair{addressToPubKeyHex(addr3, t, d) + "000370d6", txidB2T1 + "01", nil}, + keyPair{addressToPubKeyHex(addr2, t, d) + "000370d6", txidB2T1 + "03", nil}, + keyPair{addressToPubKeyHex(addr5, t, d) + "000370d6", txidB2T3 + "00" + txidB2T3 + "01", nil}, + keyPair{addressToPubKeyHex(addrA, t, d) + "000370d6", txidB2T4 + "00", nil}, + keyPair{addressToPubKeyHex(addr4, t, d) + "000370d6", txidB2T2 + "03", nil}, }); err != nil { { t.Fatal(err) } } - if err := checkColumn(d, cfUnspentTxs, []keyPair{ + if err := checkColumn(d, cfTxAddresses, []keyPair{ keyPair{ - "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", - addressToPubKeyHexWithLength("mfcWp7DB6NuaZsExybTTXpVgWz559Np4Ti", t, d) + "00", + txidB1T1, + varuintToHex(225493) + + "00" + + "02" + + addressToPubKeyHexWithLength(addr1, t, d) + bigintToHex(satB1T1A1) + + spentAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2), nil, }, keyPair{ - "7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "02", + txidB1T2, + varuintToHex(225493) + + "00" + + "03" + + spentAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + spentAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + spentAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5), nil, }, keyPair{ - "3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "02", - }) - }, + txidB2T1, + varuintToHex(225494) + + "02" + + inputAddressToPubKeyHexWithLength(addr3, t, d) + bigintToHex(satB1T2A3) + + inputAddressToPubKeyHexWithLength(addr2, t, d) + bigintToHex(satB1T1A2) + + "02" + + spentAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + addressToPubKeyHexWithLength(addr7, t, d) + bigintToHex(satB2T1A7), + nil, }, keyPair{ - "05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "00", + txidB2T2, + varuintToHex(225494) + + "02" + + inputAddressToPubKeyHexWithLength(addr6, t, d) + bigintToHex(satB2T1A6) + + inputAddressToPubKeyHexWithLength(addr4, t, d) + bigintToHex(satB1T2A4) + + "02" + + addressToPubKeyHexWithLength(addr8, t, d) + bigintToHex(satB2T2A8) + + addressToPubKeyHexWithLength(addr9, t, d) + bigintToHex(satB2T2A9), + nil, + }, + keyPair{ + txidB2T3, + varuintToHex(225494) + + "01" + + inputAddressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB1T2A5) + + "01" + + addressToPubKeyHexWithLength(addr5, t, d) + bigintToHex(satB2T3A5), + nil, + }, + keyPair{ + txidB2T4, + varuintToHex(225494) + + "01" + inputAddressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero) + + "02" + + addressToPubKeyHexWithLength(addrA, t, d) + bigintToHex(satB2T4AA) + + addressToPubKeyHexWithLength("", t, d) + bigintToHex(satZero), nil, }, }); err != nil { @@ -390,20 +560,30 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { t.Fatal(err) } } - if err := checkColumn(d, cfBlockAddresses, []keyPair{ - keyPair{"000370d6", "", - func(v string) bool { - return compareFuncBlockAddresses(t, v, []string{ - addressToPubKeyHexWithLength("mzB8cYrfRwFRFAGTDzV8LkUQy5BQicxGhX", t, d) + "00", - addressToPubKeyHexWithLength("mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL", t, d) + "00", - addressToPubKeyHexWithLength("mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", t, d) + "00", - addressToPubKeyHexWithLength("mmJx9Y8ayz9h14yd9fgCW1bUKoEpkBAquP", t, d) + "00", - addressToPubKeyHexWithLength("mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "00", - addressToPubKeyHexWithLength("mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", t, d) + "02" + "00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840" + "02", - addressToPubKeyHexWithLength("2Mz1CYoppGGsLNUGF2YDhTif6J661JitALS", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "02", - addressToPubKeyHexWithLength("2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1", t, d) + "02" + "effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75" + "04", - }) - }, + if err := checkColumn(d, cfAddressBalance, []keyPair{ + keyPair{addressToPubKeyHex(addr1, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB1T1A1), nil}, + keyPair{addressToPubKeyHex(addr2, t, d), "02" + bigintToHex(satB1T1A2) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr3, t, d), "02" + bigintToHex(satB1T2A3) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr4, t, d), "02" + bigintToHex(satB1T2A4) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr5, t, d), "02" + bigintToHex(satB1T2A5) + bigintToHex(satB2T3A5), nil}, + keyPair{addressToPubKeyHex(addr6, t, d), "02" + bigintToHex(satB2T1A6) + bigintToHex(satZero), nil}, + keyPair{addressToPubKeyHex(addr7, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T1A7), nil}, + keyPair{addressToPubKeyHex(addr8, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A8), nil}, + keyPair{addressToPubKeyHex(addr9, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T2A9), nil}, + keyPair{addressToPubKeyHex(addrA, t, d), "01" + bigintToHex(satZero) + bigintToHex(satB2T4AA), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + if err := checkColumn(d, cfBlockTxs, []keyPair{ + keyPair{ + "000370d6", + txidB2T1 + "02" + txidB1T2 + "00" + txidB1T1 + "02" + + txidB2T2 + "02" + txidB2T1 + "00" + txidB1T2 + "02" + + txidB2T3 + "01" + txidB1T2 + "04" + + txidB2T4 + "01" + "0000000000000000000000000000000000000000000000000000000000000000" + "00", + nil, }, }); err != nil { { @@ -474,19 +654,16 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality // 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using blockaddresses column -// 7) Reconnect block 2 and disconnect blocks 1 and 2 using full scan - expect error +// 6) Disconnect the block 2 using BlockTxs column +// 7) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure func TestRocksDB_Index_UTXO(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ - BitcoinParser: &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{BlockAddressesToKeep: 1}, - Params: btc.GetChainParams("test"), - }, + BitcoinParser: bitcoinTestnetParser(), }) defer closeAndDestroyRocksDB(t, d) - // connect 1st block - will log warnings about missing UTXO transactions in cfUnspentTxs column + // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column block1 := getTestUTXOBlock1(t, d) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) @@ -501,19 +678,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { verifyAfterUTXOBlock2(t, d) // get transactions for various addresses / low-high ranges - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225493, 225493, []txidVoutOutput{ - txidVoutOutput{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1, true}, + verifyGetTransactions(t, d, addr2, 225493, 225493, []txidVoutOutput{ + txidVoutOutput{txidB1T1, 1, true}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 225494, 1000000, []txidVoutOutput{ - txidVoutOutput{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 1, false}, + verifyGetTransactions(t, d, addr2, 225494, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T1, 1, false}, }, nil) - verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eEtz", 500000, 1000000, []txidVoutOutput{}, nil) - verifyGetTransactions(t, d, "mwwoKQE5Lb1G4picHSHDQKg8jw424PF9SC", 0, 1000000, []txidVoutOutput{ - txidVoutOutput{"3d90d15ed026dc45e19ffb52875ed18fa9e8012ad123d7f7212176e2b0ebdb71", 0, true}, + verifyGetTransactions(t, d, addr2, 500000, 1000000, []txidVoutOutput{}, nil) + verifyGetTransactions(t, d, addr8, 0, 1000000, []txidVoutOutput{ + txidVoutOutput{txidB2T2, 0, true}, }, nil) verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("checksum mismatch")) @@ -538,6 +715,31 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatalf("GetBlockHash: got hash %v, expected %v", hash, "0000000076fbbed90fd75b0e18856aa35baa984e9c9d444cf746ad85e94e2997") } + // Not connected block + hash, err = d.GetBlockHash(225495) + if err != nil { + t.Fatal(err) + } + if hash != "" { + t.Fatalf("GetBlockHash: got hash '%v', expected ''", hash) + } + + // GetBlockHash + info, err := d.GetBlockInfo(225494) + if err != nil { + t.Fatal(err) + } + iw := &BlockInfo{ + Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Txs: 4, + Size: 2345678, + Time: 1534859123, + Height: 225494, + } + if !reflect.DeepEqual(info, iw) { + t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) + } + // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock testTxCache(t, d, block1, &block1.Txs[0]) testTxCache(t, d, block2, &block2.Txs[0]) @@ -561,13 +763,19 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } verifyAfterUTXOBlock2(t, d) + // try to disconnect both blocks, however only the last one is kept, it is not possible + err = d.DisconnectBlockRangeUTXO(225493, 225494) + if err == nil || err.Error() != "Cannot disconnect blocks with height 225493 and lower. It is necessary to rebuild index." { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs // and that the cached tx is removed - err = d.DisconnectBlockRange(225494, 225494) + err = d.DisconnectBlockRangeUTXO(225494, 225494) if err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, true) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { @@ -575,150 +783,337 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } } + // connect block again and verify the state of db + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterUTXOBlock2(t, d) + + // test public methods for address balance and tx addresses + + ab, err := d.GetAddressBalance(addr5) + if err != nil { + t.Fatal(err) + } + abw := &AddrBalance{ + Txs: 2, + SentSat: *satB1T2A5, + BalanceSat: *satB2T3A5, + } + if !reflect.DeepEqual(ab, abw) { + t.Errorf("GetAddressBalance() = %+v, want %+v", ab, abw) + } + rs := ab.ReceivedSat() + rsw := satB1T2A5.Add(satB1T2A5, satB2T3A5) + if rs.Cmp(rsw) != 0 { + t.Errorf("GetAddressBalance().ReceivedSat() = %v, want %v", rs, rsw) + } + + ta, err := d.GetTxAddresses(txidB2T1) + if err != nil { + t.Fatal(err) + } + taw := &TxAddresses{ + Height: 225494, + Inputs: []TxInput{ + { + AddrDesc: addressToAddrDesc(addr3, d.chainParser), + ValueSat: *satB1T2A3, + }, + { + AddrDesc: addressToAddrDesc(addr2, d.chainParser), + ValueSat: *satB1T1A2, + }, + }, + Outputs: []TxOutput{ + { + AddrDesc: addressToAddrDesc(addr6, d.chainParser), + Spent: true, + ValueSat: *satB2T1A6, + }, + { + AddrDesc: addressToAddrDesc(addr7, d.chainParser), + Spent: false, + ValueSat: *satB2T1A7, + }, + }, + } + if !reflect.DeepEqual(ta, taw) { + t.Errorf("GetTxAddresses() = %+v, want %+v", ta, taw) + } + ia, _, err := ta.Inputs[0].Addresses(d.chainParser) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(ia, []string{addr3}) { + t.Errorf("GetTxAddresses().Inputs[0].Addresses() = %v, want %v", ia, []string{addr3}) + } + } -func Test_findAndRemoveUnspentAddr(t *testing.T) { - type args struct { - unspentAddrs string - vout uint32 +func Test_BulkConnect_UTXO(t *testing.T) { + d := setupRocksDB(t, &testBitcoinParser{ + BitcoinParser: bitcoinTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + bc, err := d.InitBulkConnect() + if err != nil { + t.Fatal(err) } + + if d.is.DbState != common.DbStateInconsistent { + t.Fatal("DB not in DbStateInconsistent") + } + + if err := bc.ConnectBlock(getTestUTXOBlock1(t, d), false); err != nil { + t.Fatal(err) + } + if err := checkColumn(d, cfBlockTxs, []keyPair{}); err != nil { + { + t.Fatal(err) + } + } + + if err := bc.ConnectBlock(getTestUTXOBlock2(t, d), true); err != nil { + t.Fatal(err) + } + + if err := bc.Close(); err != nil { + t.Fatal(err) + } + + if d.is.DbState != common.DbStateOpen { + t.Fatal("DB not in DbStateOpen") + } + + verifyAfterUTXOBlock2(t, d) +} + +func Test_packBigint_unpackBigint(t *testing.T) { + bigbig1, _ := big.NewInt(0).SetString("123456789123456789012345", 10) + bigbig2, _ := big.NewInt(0).SetString("12345678912345678901234512389012345123456789123456789012345123456789123456789012345", 10) + bigbigbig := big.NewInt(0) + bigbigbig.Mul(bigbig2, bigbig2) + bigbigbig.Mul(bigbigbig, bigbigbig) + bigbigbig.Mul(bigbigbig, bigbigbig) tests := []struct { - name string - args args - want string - want2 string + name string + bi *big.Int + buf []byte + toobiglen int }{ { - name: "3", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 3, - }, - want: "64635167006868", - want2: "029c0010517a0115887452870212709393588893935687040e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", + name: "0", + bi: big.NewInt(0), + buf: make([]byte, maxPackedBigintBytes), }, { - name: "10", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 10, - }, - want: "61", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112", + name: "1", + bi: big.NewInt(1), + buf: make([]byte, maxPackedBigintBytes), }, { - name: "not there", - args: args{ - unspentAddrs: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", - vout: 11, - }, - want: "", - want2: "029c0010517a0115887452870212709393588893935687040e64635167006868060e76519351880087080a7b7b0115870a3276a9144150837fb91d9461d6b95059842ab85262c2923f88ac0c08636751680e04578710029112026114", + name: "54321", + bi: big.NewInt(54321), + buf: make([]byte, 249), + }, + { + name: "12345678", + bi: big.NewInt(12345678), + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "123456789123456789", + bi: big.NewInt(123456789123456789), + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "bigbig1", + bi: bigbig1, + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "bigbig2", + bi: bigbig2, + buf: make([]byte, maxPackedBigintBytes), + }, + { + name: "bigbigbig", + bi: bigbigbig, + buf: make([]byte, maxPackedBigintBytes), + toobiglen: 242, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.unspentAddrs) - if err != nil { - panic(err) - } - got, got2 := findAndRemoveUnspentAddr(b, tt.args.vout) - h := hex.EncodeToString(got) - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("findAndRemoveUnspentAddr() got = %v, want %v", h, tt.want) - } - h2 := hex.EncodeToString(got2) - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("findAndRemoveUnspentAddr() got2 = %v, want %v", h2, tt.want2) + // packBigint + got := packBigint(tt.bi, tt.buf) + if tt.toobiglen == 0 { + // create buffer that we expect + bb := tt.bi.Bytes() + want := append([]byte(nil), byte(len(bb))) + want = append(want, bb...) + if got != len(want) { + t.Errorf("packBigint() = %v, want %v", got, len(want)) + } + for i := 0; i < got; i++ { + if tt.buf[i] != want[i] { + t.Errorf("packBigint() buf = %v, want %v", tt.buf[:got], want) + break + } + } + // unpackBigint + got1, got2 := unpackBigint(tt.buf) + if got2 != len(want) { + t.Errorf("unpackBigint() = %v, want %v", got2, len(want)) + } + if tt.bi.Cmp(&got1) != 0 { + t.Errorf("unpackBigint() = %v, want %v", got1, tt.bi) + } + } else { + if got != tt.toobiglen { + t.Errorf("packBigint() = %v, want toobiglen %v", got, tt.toobiglen) + } } }) } } -type hexoutpoint struct { - txID string - vout int32 +func addressToAddrDesc(addr string, parser bchain.BlockChainParser) []byte { + b, err := parser.GetAddrDescFromAddress(addr) + if err != nil { + panic(err) + } + return b } -func Test_unpackBlockAddresses(t *testing.T) { - d := setupRocksDB(t, &testBitcoinParser{BitcoinParser: &btc.BitcoinParser{Params: btc.GetChainParams("test")}}) - defer closeAndDestroyRocksDB(t, d) - type args struct { - buf string - } +func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { + parser := bitcoinTestnetParser() tests := []struct { - name string - args args - want []string - want2 [][]hexoutpoint - wantErr bool + name string + hex string + data *TxAddresses }{ { name: "1", - args: args{"029c0010517a011588745287047c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d250000b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa38400612709393588893935687000e64635167006868000e7651935188008702effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac7502"}, - want: []string{"9c", "517a011588745287", "709393588893935687", "64635167006868", "76519351880087"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25", 0}, - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 3}, + hex: "7b0216001443aac20a116e09ea4f7914be1c55e4c17aa600b70016001454633aa8bd2e552bd4e89c01e73c1b7905eb58460811207cb68a199872012d001443aac20a116e09ea4f7914be1c55e4c17aa600b70101", + data: &TxAddresses{ + Height: 123, + Inputs: []TxInput{ + { + AddrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + ValueSat: *big.NewInt(0), + }, + { + AddrDesc: addressToAddrDesc("tb1q233n429a9e2jh48gnsq7w0qm0yz7kkzx0qczw8", parser), + ValueSat: *big.NewInt(1234123421342341234), + }, }, - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + Outputs: []TxOutput{ + { + AddrDesc: addressToAddrDesc("tb1qgw4vyzs3dcy75nmezjlpc40yc9a2vq9hghdyt2", parser), + ValueSat: *big.NewInt(1), + Spent: true, + }, }, }, }, { - name: "1", - args: args{"3276A914B434EB0C1A3B7A02E8A29CC616E791EF1E0BF51F88AC003276A9143F8BA3FDA3BA7B69F5818086E12223C6DD25E3C888AC003276A914A08EAE93007F22668AB5E4A9C83C8CD1C325E3E088AC02EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75003276A9148BDF0AA3C567AA5975C2E61321B8BEBBE7293DF688AC0200B2C06055E5E90E9C82BD4181FDE310104391A7FA4F289B1704E5D90CAA3840022EA9144A21DB08FB6882CB152E1FF06780A430740F77048702EFFD9EF509383D536B1C8AF5BF434C8EFBF521A4F2BEFD4022BBD68694B4AC75023276A914CCAAAF374E1B06CB83118453D102587B4273D09588AC003276A9148D802C045445DF49613F6A70DDD2E48526F3701F88AC00"}, - want: []string{"76a914b434eb0c1a3b7a02e8a29cc616e791ef1e0bf51f88ac", "76a9143f8ba3fda3ba7b69f5818086e12223c6dd25e3c888ac", "76a914a08eae93007f22668ab5e4a9c83c8cd1c325e3e088ac", "76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac", "a9144a21db08fb6882cb152e1ff06780a430740f770487", "76a914ccaaaf374e1b06cb83118453d102587b4273d09588ac", "76a9148d802c045445df49613f6a70ddd2e48526f3701f88ac"}, - want2: [][]hexoutpoint{ - []hexoutpoint{}, - []hexoutpoint{}, - []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 0}, + name: "2", + hex: "e0390317a9149eb21980dc9d413d8eac27314938b9da920ee53e8705021918f2c017a91409f70b896169c37981d2b54b371df0d81a136a2c870501dd7e28c017a914e371782582a4addb541362c55565d2cdf56f6498870501a1e35ec0052fa9141d9ca71efa36d814424ea6ca1437e67287aebe348705012aadcac02ea91424fbc77cdc62702ade74dcf989c15e5d3f9240bc870501664894c02fa914afbfb74ee994c7d45f6698738bc4226d065266f7870501a1e35ec03276a914d2a37ce20ac9ec4f15dd05a7c6e8e9fbdb99850e88ac043b9943603376a9146b2044146a4438e6e5bfbc65f147afeb64d14fbb88ac05012a05f200", + data: &TxAddresses{ + Height: 12345, + Inputs: []TxInput{ + { + AddrDesc: addressToAddrDesc("2N7iL7AvS4LViugwsdjTB13uN4T7XhV1bCP", parser), + ValueSat: *big.NewInt(9011000000), + }, + { + AddrDesc: addressToAddrDesc("2Mt9v216YiNBAzobeNEzd4FQweHrGyuRHze", parser), + ValueSat: *big.NewInt(8011000000), + }, + { + AddrDesc: addressToAddrDesc("2NDyqJpHvHnqNtL1F9xAeCWMAW8WLJmEMyD", parser), + ValueSat: *big.NewInt(7011000000), + }, }, - []hexoutpoint{ - hexoutpoint{"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840", 1}, + Outputs: []TxOutput{ + { + AddrDesc: addressToAddrDesc("2MuwoFGwABMakU7DCpdGDAKzyj2nTyRagDP", parser), + ValueSat: *big.NewInt(5011000000), + Spent: true, + }, + { + AddrDesc: addressToAddrDesc("2Mvcmw7qkGXNWzkfH1EjvxDcNRGL1Kf2tEM", parser), + ValueSat: *big.NewInt(6011000000), + }, + { + AddrDesc: addressToAddrDesc("2N9GVuX3XJGHS5MCdgn97gVezc6EgvzikTB", parser), + ValueSat: *big.NewInt(7011000000), + Spent: true, + }, + { + AddrDesc: addressToAddrDesc("mzii3fuRSpExMLJEHdHveW8NmiX8MPgavk", parser), + ValueSat: *big.NewInt(999900000), + }, + { + AddrDesc: addressToAddrDesc("mqHPFTRk23JZm9W1ANuEFtwTYwxjESSgKs", parser), + ValueSat: *big.NewInt(5000000000), + Spent: true, + }, }, - []hexoutpoint{ - hexoutpoint{"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75", 1}, + }, + }, + { + name: "empty address", + hex: "baef9a1501000204d2020002162e010162", + data: &TxAddresses{ + Height: 123456789, + Inputs: []TxInput{ + { + AddrDesc: []byte{}, + ValueSat: *big.NewInt(1234), + }, }, - []hexoutpoint{}, - []hexoutpoint{}, + Outputs: []TxOutput{ + { + AddrDesc: []byte{}, + ValueSat: *big.NewInt(5678), + }, + { + AddrDesc: []byte{}, + ValueSat: *big.NewInt(98), + Spent: true, + }, + }, + }, + }, + { + name: "empty", + hex: "000000", + data: &TxAddresses{ + Inputs: []TxInput{}, + Outputs: []TxOutput{}, }, }, } + varBuf := make([]byte, maxPackedBigintBytes) + buf := make([]byte, 1024) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b, err := hex.DecodeString(tt.args.buf) - if err != nil { - panic(err) + b := packTxAddresses(tt.data, buf, varBuf) + hex := hex.EncodeToString(b) + if !reflect.DeepEqual(hex, tt.hex) { + t.Errorf("packTxAddresses() = %v, want %v", hex, tt.hex) } - got, got2, err := d.unpackBlockAddresses(b) - if (err != nil) != tt.wantErr { - t.Errorf("unpackBlockAddresses() error = %v, wantErr %v", err, tt.wantErr) + got1, err := unpackTxAddresses(b) + if err != nil { + t.Errorf("unpackTxAddresses() error = %v", err) return } - h := make([]string, len(got)) - for i, g := range got { - h[i] = hex.EncodeToString(g) - } - if !reflect.DeepEqual(h, tt.want) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h, tt.want) - } - h2 := make([][]hexoutpoint, len(got2)) - for i, g := range got2 { - ho := make([]hexoutpoint, len(g)) - for j, o := range g { - ho[j] = hexoutpoint{hex.EncodeToString(o.btxID), o.vout} - } - h2[i] = ho - } - if !reflect.DeepEqual(h2, tt.want2) { - t.Errorf("unpackBlockAddresses() = %v, want %v", h2, tt.want2) + if !reflect.DeepEqual(got1, tt.data) { + t.Errorf("unpackTxAddresses() = %+v, want %+v", got1, tt.data) } }) } diff --git a/db/sync.go b/db/sync.go index 6c2e281e..468c2fca 100644 --- a/db/sync.go +++ b/db/sync.go @@ -67,6 +67,7 @@ func (w *SyncWorker) ResyncIndex(onNewBlock bchain.OnNewBlockFunc) error { case errSynced: // this is not actually error but flag that resync wasn't necessary w.is.FinishedSyncNoChange() + w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) return nil } @@ -203,26 +204,46 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { } var err error var wg sync.WaitGroup - bch := make(chan *bchain.Block, w.syncWorkers) + bch := make([]chan *bchain.Block, w.syncWorkers) + for i := 0; i < w.syncWorkers; i++ { + bch[i] = make(chan *bchain.Block) + } hch := make(chan hashHeight, w.syncWorkers) hchClosed := atomic.Value{} hchClosed.Store(false) - var getBlockMux sync.Mutex - getBlockCond := sync.NewCond(&getBlockMux) - lastConnectedBlock := lower - 1 writeBlockDone := make(chan struct{}) + terminating := make(chan struct{}) writeBlockWorker := func() { defer close(writeBlockDone) + bc, err := w.db.InitBulkConnect() + if err != nil { + glog.Error("sync: InitBulkConnect error ", err) + } lastBlock := lower - 1 - for b := range bch { - if lastBlock+1 != b.Height { - glog.Error("writeBlockWorker skipped block, last connected block", lastBlock, ", new block ", b.Height) + keep := uint32(w.chain.GetChainParser().KeepBlockAddresses()) + WriteBlockLoop: + for { + select { + case b := <-bch[(lastBlock+1)%uint32(w.syncWorkers)]: + if b == nil { + // channel is closed and empty - work is done + break WriteBlockLoop + } + if b.Height != lastBlock+1 { + glog.Fatal("writeBlockWorker skipped block, expected block ", lastBlock+1, ", new block ", b.Height) + } + err := bc.ConnectBlock(b, b.Height+keep > higher) + if err != nil { + glog.Fatal("writeBlockWorker ", b.Height, " ", b.Hash, " error ", err) + } + lastBlock = b.Height + case <-terminating: + break WriteBlockLoop } - err := w.db.ConnectBlock(b) - if err != nil { - glog.Error("writeBlockWorker ", b.Height, " ", b.Hash, " error ", err) - } - lastBlock = b.Height + } + err = bc.Close() + if err != nil { + glog.Error("sync: bulkconnect.Close error ", err) } glog.Info("WriteBlock exiting...") } @@ -230,6 +251,7 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { defer wg.Done() var err error var block *bchain.Block + GetBlockLoop: for hh := range hch { for { block, err = w.chain.GetBlock(hh.hash, hh.height) @@ -249,24 +271,11 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { if w.dryRun { continue } - getBlockMux.Lock() - for { - // we must make sure that the blocks are written to db in the correct order - if lastConnectedBlock+1 == hh.height { - // we have the right block, pass it to the writeBlockWorker - lastConnectedBlock = hh.height - bch <- block - getBlockCond.Broadcast() - break - } - // break the endless loop on OS signal - if hchClosed.Load() == true { - break - } - // wait for the time this block is top be passed to the writeBlockWorker - getBlockCond.Wait() + select { + case bch[hh.height%uint32(w.syncWorkers)] <- block: + case <-terminating: + break GetBlockLoop } - getBlockMux.Unlock() } glog.Info("getBlockWorker ", i, " exiting...") } @@ -276,11 +285,15 @@ func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { } go writeBlockWorker() var hash string + start := time.Now() + msTime := time.Now().Add(1 * time.Minute) ConnectLoop: for h := lower; h <= higher; { select { case <-w.chanOsSignal: err = errors.Errorf("connectBlocksParallel interrupted at height %d", h) + // signal all workers to terminate their loops (error loops are interrupted below) + close(terminating) break ConnectLoop default: hash, err = w.chain.GetBlockHash(h) @@ -292,22 +305,25 @@ ConnectLoop: } hch <- hashHeight{hash, h} if h > 0 && h%1000 == 0 { - glog.Info("connecting block ", h, " ", hash) + glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start), " ", w.db.GetAndResetConnectBlockStats()) + start = time.Now() + } + if msTime.Before(time.Now()) { + glog.Info(w.db.GetMemoryStats()) + w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) + msTime = time.Now().Add(10 * time.Minute) } h++ } } close(hch) - // signal stop to workers that are in a loop + // signal stop to workers that are in a error loop hchClosed.Store(true) - // broadcast syncWorkers times to unstuck all waiting getBlockWorkers - for i := 0; i < w.syncWorkers; i++ { - getBlockCond.Broadcast() - } - // first wait for the getBlockWorkers to finish and then close bch channel - // so that the getBlockWorkers do not write to the closed channel + // wait for workers and close bch that will stop writer loop wg.Wait() - close(bch) + for i := 0; i < w.syncWorkers; i++ { + close(bch[i]) + } <-writeBlockDone return err } @@ -346,25 +362,23 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { } // DisconnectBlocks removes all data belonging to blocks in range lower-higher, -// using block data from blockchain, if they are available, -// otherwise doing full scan func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) - // if the chain uses Block to Addresses mapping, always use DisconnectBlockRange - if w.chain.GetChainParser().KeepBlockAddresses() > 0 { - return w.db.DisconnectBlockRange(lower, higher) + // if the chain is UTXO, always use DisconnectBlockRange + if w.chain.GetChainParser().IsUTXOChain() { + return w.db.DisconnectBlockRangeUTXO(lower, higher) } blocks := make([]*bchain.Block, len(hashes)) var err error - // get all blocks first to see if we can avoid full scan + // try to get all blocks first to see if we can avoid full scan for i, hash := range hashes { blocks[i], err = w.chain.GetBlock(hash, 0) if err != nil { // cannot get a block, we must do full range scan - return w.db.DisconnectBlockRange(lower, higher) + return w.db.DisconnectBlockRangeNonUTXO(lower, higher) } } - // then disconnect one after another + // got all blocks to be disconnected, disconnect them one after another for i, block := range blocks { glog.Info("Disconnecting block ", (int(higher) - i), " ", block.Hash) if err = w.db.DisconnectBlock(block); err != nil { diff --git a/docs/config.md b/docs/config.md index af3f5cd0..61e76710 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,7 +79,7 @@ Good examples of coin configuration are * `system_user` – User used to run Blockbook service. See convention note in [build guide](/docs/build.md#on-naming-conventions-and-versioning). * `internal_binding_template` – Template for *-internal* parameter. See note on templates below. * `public_binding_template` – Template for *-public* parameter. See note on templates below. - * `explorer_url` – URL of blockchain explorer. + * `explorer_url` – URL of blockchain explorer. Leave empty for internal explorer. * `additional_params` – Additional params of exec command (see [Dogecoin definition](configs/coins/dogecoin.json)). * `block_chain` – Configuration of BlockChain type that ensures communication with back-end service. All options must be tweaked for each individual coin separely. diff --git a/docs/ports.md b/docs/ports.md index 77072825..e400f01d 100644 --- a/docs/ports.md +++ b/docs/ports.md @@ -8,7 +8,7 @@ | Dash | 9033 | 9133 | 8033 | 38333 | | Litecoin | 9034 | 9134 | 8034 | 38334 | | Bgold | 9035 | 9135 | 8035 | 38335 | -| Ethereum | 9036 | 9136 | 8036 | 38336 p2p, 8136 http | +| Ethereum | 9036 | 9136 | 8036 | 8136 http, 38336 p2p | | Ethereum Classic | 9037 | 9137 | 8037 | | | Dogecoin | 9038 | 9138 | 8038 | 38338 | | Namecoin | 9039 | 9139 | 8039 | 38339 | diff --git a/server/internal.go b/server/internal.go index a4752b86..5e12dcf1 100644 --- a/server/internal.go +++ b/server/internal.go @@ -1,20 +1,17 @@ package server import ( + "blockbook/api" "blockbook/bchain" "blockbook/common" "blockbook/db" "context" "encoding/json" - "errors" "fmt" "net/http" - "strconv" - "time" "github.com/golang/glog" - "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -27,29 +24,21 @@ type InternalServer struct { chain bchain.BlockChain chainParser bchain.BlockChainParser is *common.InternalState -} - -type resAboutBlockbookInternal struct { - Coin string `json:"coin"` - Host string `json:"host"` - Version string `json:"version"` - GitCommit string `json:"gitcommit"` - BuildTime string `json:"buildtime"` - InSync bool `json:"inSync"` - BestHeight uint32 `json:"bestHeight"` - LastBlockTime time.Time `json:"lastBlockTime"` - InSyncMempool bool `json:"inSyncMempool"` - LastMempoolTime time.Time `json:"lastMempoolTime"` - MempoolSize int `json:"mempoolSize"` - DbColumns []common.InternalStateColumn `json:"dbColumns"` + api *api.Worker } // NewInternalServer creates new internal http interface to blockbook and returns its handle -func NewInternalServer(httpServerBinding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState) (*InternalServer, error) { - r := mux.NewRouter() +func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState) (*InternalServer, error) { + api, err := api.NewWorker(db, chain, txCache, is) + if err != nil { + return nil, err + } + + addr, path := splitBinding(binding) + serveMux := http.NewServeMux() https := &http.Server{ - Addr: httpServerBinding, - Handler: r, + Addr: addr, + Handler: serveMux, } s := &InternalServer{ https: https, @@ -59,15 +48,12 @@ func NewInternalServer(httpServerBinding string, certFiles string, db *db.RocksD chain: chain, chainParser: chain.GetChainParser(), is: is, + api: api, } - r.HandleFunc("/", s.index) - r.HandleFunc("/bestBlockHash", s.bestBlockHash) - r.HandleFunc("/blockHash/{height}", s.blockHash) - r.HandleFunc("/transactions/{address}/{lower}/{higher}", s.transactions) - r.HandleFunc("/confirmedTransactions/{address}/{lower}/{higher}", s.confirmedTransactions) - r.HandleFunc("/unconfirmedTransactions/{address}", s.unconfirmedTransactions) - r.HandleFunc("/metrics", promhttp.Handler().ServeHTTP) + serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) + serveMux.HandleFunc(path+"metrics", promhttp.Handler().ServeHTTP) + serveMux.HandleFunc(path, s.index) return s, nil } @@ -94,142 +80,11 @@ func (s *InternalServer) Shutdown(ctx context.Context) error { return s.https.Shutdown(ctx) } -func respondError(w http.ResponseWriter, err error, context string) { - w.WriteHeader(http.StatusBadRequest) - glog.Errorf("internal server: (context %s) error: %v", context, err) -} - -func respondHashData(w http.ResponseWriter, hash string) { - type hashData struct { - Hash string `json:"hash"` - } - json.NewEncoder(w).Encode(hashData{ - Hash: hash, - }) -} - func (s *InternalServer) index(w http.ResponseWriter, r *http.Request) { - vi := common.GetVersionInfo() - ss, bh, st := s.is.GetSyncState() - ms, mt, msz := s.is.GetMempoolSyncState() - a := resAboutBlockbookInternal{ - Coin: s.is.Coin, - Host: s.is.Host, - Version: vi.Version, - GitCommit: vi.GitCommit, - BuildTime: vi.BuildTime, - InSync: ss, - BestHeight: bh, - LastBlockTime: st, - InSyncMempool: ms, - LastMempoolTime: mt, - MempoolSize: msz, - DbColumns: s.is.GetAllDBColumnStats(), - } - buf, err := json.MarshalIndent(a, "", " ") + si, err := s.api.GetSystemInfo(true) + buf, err := json.MarshalIndent(si, "", " ") if err != nil { glog.Error(err) } w.Write(buf) } - -func (s *InternalServer) bestBlockHash(w http.ResponseWriter, r *http.Request) { - _, hash, err := s.db.GetBestBlock() - if err != nil { - respondError(w, err, "bestBlockHash") - return - } - respondHashData(w, hash) -} - -func (s *InternalServer) blockHash(w http.ResponseWriter, r *http.Request) { - heightString := mux.Vars(r)["height"] - var hash string - height, err := strconv.ParseUint(heightString, 10, 32) - if err == nil { - hash, err = s.db.GetBlockHash(uint32(height)) - } - if err != nil { - respondError(w, err, fmt.Sprintf("blockHash %s", heightString)) - } else { - respondHashData(w, hash) - } -} - -func (s *InternalServer) getAddress(r *http.Request) (address string, err error) { - address, ok := mux.Vars(r)["address"] - if !ok { - err = errors.New("Empty address") - } - return -} - -func (s *InternalServer) getAddressAndHeightRange(r *http.Request) (address string, lower, higher uint32, err error) { - address, err = s.getAddress(r) - if err != nil { - return - } - higher64, err := strconv.ParseUint(mux.Vars(r)["higher"], 10, 32) - if err != nil { - return - } - lower64, err := strconv.ParseUint(mux.Vars(r)["lower"], 10, 32) - if err != nil { - return - } - return address, uint32(lower64), uint32(higher64), err -} - -type transactionList struct { - Txid []string `json:"txid"` -} - -func (s *InternalServer) unconfirmedTransactions(w http.ResponseWriter, r *http.Request) { - address, err := s.getAddress(r) - if err != nil { - respondError(w, err, fmt.Sprint("unconfirmedTransactions for address", address)) - } - txs, err := s.chain.GetMempoolTransactions(address) - if err != nil { - respondError(w, err, fmt.Sprint("unconfirmedTransactions for address", address)) - } - txList := transactionList{Txid: txs} - json.NewEncoder(w).Encode(txList) -} - -func (s *InternalServer) confirmedTransactions(w http.ResponseWriter, r *http.Request) { - address, lower, higher, err := s.getAddressAndHeightRange(r) - if err != nil { - respondError(w, err, fmt.Sprint("confirmedTransactions for address", address)) - } - txList := transactionList{} - err = s.db.GetTransactions(address, lower, higher, func(txid string, vout uint32, isOutput bool) error { - txList.Txid = append(txList.Txid, txid) - return nil - }) - if err != nil { - respondError(w, err, fmt.Sprint("confirmedTransactions for address", address)) - } - json.NewEncoder(w).Encode(txList) -} - -func (s *InternalServer) transactions(w http.ResponseWriter, r *http.Request) { - address, lower, higher, err := s.getAddressAndHeightRange(r) - if err != nil { - respondError(w, err, fmt.Sprint("transactions for address", address)) - } - txList := transactionList{} - err = s.db.GetTransactions(address, lower, higher, func(txid string, vout uint32, isOutput bool) error { - txList.Txid = append(txList.Txid, txid) - return nil - }) - if err != nil { - respondError(w, err, fmt.Sprint("transactions for address", address)) - } - txs, err := s.chain.GetMempoolTransactions(address) - if err != nil { - respondError(w, err, fmt.Sprint("transactions for address", address)) - } - txList.Txid = append(txList.Txid, txs...) - json.NewEncoder(w).Encode(txList) -} diff --git a/server/public.go b/server/public.go index d208a3de..b39aa796 100644 --- a/server/public.go +++ b/server/public.go @@ -10,6 +10,8 @@ import ( "fmt" "html/template" "net/http" + "reflect" + "runtime" "strconv" "strings" "time" @@ -17,28 +19,32 @@ import ( "github.com/golang/glog" ) -const blockbookAbout = "Blockbook - blockchain indexer for TREZOR wallet https://trezor.io/. Do not use for any other purpose." +const txsOnPage = 25 +const blocksOnPage = 50 +const txsInAPI = 1000 -// PublicServer is handle to public http server +// PublicServer is a handle to public http server type PublicServer struct { - binding string - certFiles string - socketio *SocketIoServer - https *http.Server - db *db.RocksDB - txCache *db.TxCache - chain bchain.BlockChain - chainParser bchain.BlockChainParser - api *api.Worker - explorerURL string - metrics *common.Metrics - is *common.InternalState - txTpl *template.Template - addressTpl *template.Template + binding string + certFiles string + socketio *SocketIoServer + https *http.Server + db *db.RocksDB + txCache *db.TxCache + chain bchain.BlockChain + chainParser bchain.BlockChainParser + api *api.Worker + explorerURL string + internalExplorer bool + metrics *common.Metrics + is *common.InternalState + templates []*template.Template + debug bool } -// NewPublicServerS creates new public server http interface to blockbook and returns its handle -func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState) (*PublicServer, error) { +// NewPublicServer creates new public server http interface to blockbook and returns its handle +// only basic functionality is mapped, to map all functions, call +func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState, debugMode bool) (*PublicServer, error) { api, err := api.NewWorker(db, chain, txCache, is) if err != nil { @@ -58,65 +64,34 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch } s := &PublicServer{ - binding: binding, - certFiles: certFiles, - https: https, - api: api, - socketio: socketio, - db: db, - txCache: txCache, - chain: chain, - chainParser: chain.GetChainParser(), - explorerURL: explorerURL, - metrics: metrics, - is: is, + binding: binding, + certFiles: certFiles, + https: https, + api: api, + socketio: socketio, + db: db, + txCache: txCache, + chain: chain, + chainParser: chain.GetChainParser(), + explorerURL: explorerURL, + internalExplorer: explorerURL == "", + metrics: metrics, + is: is, + debug: debugMode, } + s.templates = parseTemplates() - // favicon + // map only basic functions, the rest is enabled by method MapFullPublicInterface serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) - // support for tests of socket.io interface - serveMux.Handle(path+"test.html", http.FileServer(http.Dir("./static/"))) - // redirect to Bitcore for details of transaction - serveMux.HandleFunc(path+"tx/", s.txRedirect) - serveMux.HandleFunc(path+"address/", s.addressRedirect) - // explorer - serveMux.HandleFunc(path+"explorer/tx/", s.explorerTx) - serveMux.HandleFunc(path+"explorer/address/", s.explorerAddress) serveMux.Handle(path+"static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))) - // API calls - serveMux.HandleFunc(path+"api/block-index/", s.apiBlockIndex) - serveMux.HandleFunc(path+"api/tx/", s.apiTx) - serveMux.HandleFunc(path+"api/address/", s.apiAddress) - // handle socket.io - serveMux.Handle(path+"socket.io/", socketio.GetHandler()) // default handler - serveMux.HandleFunc(path, s.index) - - s.txTpl, s.addressTpl = parseTemplates() + serveMux.HandleFunc(path, s.htmlTemplateHandler(s.explorerIndex)) + // default API handler + serveMux.HandleFunc(path+"api/", s.jsonHandler(s.apiIndex)) return s, nil } -func parseTemplates() (txTpl, addressTpl *template.Template) { - templateFuncMap := template.FuncMap{ - "formatUnixTime": formatUnixTime, - "formatAmount": formatAmount, - "setTxToTemplateData": setTxToTemplateData, - "stringInSlice": stringInSlice, - } - txTpl = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) - addressTpl = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/base.html")) - return -} - -func formatUnixTime(ut int64) string { - return time.Unix(ut, 0).Format(time.RFC1123) -} - -func formatAmount(a float64) string { - return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%0.8f", a), "0"), ".") -} - // Run starts the server func (s *PublicServer) Run() error { if s.certFiles == "" { @@ -127,6 +102,34 @@ func (s *PublicServer) Run() error { return s.https.ListenAndServeTLS(fmt.Sprint(s.certFiles, ".crt"), fmt.Sprint(s.certFiles, ".key")) } +// ConnectFullPublicInterface enables complete public functionality +func (s *PublicServer) ConnectFullPublicInterface() { + serveMux := s.https.Handler.(*http.ServeMux) + _, path := splitBinding(s.binding) + // support for tests of socket.io interface + serveMux.Handle(path+"test.html", http.FileServer(http.Dir("./static/"))) + if s.internalExplorer { + // internal explorer handlers + serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx)) + serveMux.HandleFunc(path+"address/", s.htmlTemplateHandler(s.explorerAddress)) + serveMux.HandleFunc(path+"search/", s.htmlTemplateHandler(s.explorerSearch)) + serveMux.HandleFunc(path+"blocks", s.htmlTemplateHandler(s.explorerBlocks)) + serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock)) + serveMux.HandleFunc(path+"spending/", s.htmlTemplateHandler(s.explorerSpendingTx)) + } else { + // redirect to wallet requests for tx and address, possibly to external site + serveMux.HandleFunc(path+"tx/", s.txRedirect) + serveMux.HandleFunc(path+"address/", s.addressRedirect) + } + // API calls + serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex)) + serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx)) + serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress)) + serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock)) + // socket.io interface + serveMux.Handle(path+"socket.io/", s.socketio.GetHandler()) +} + // Close closes the server func (s *PublicServer) Close() error { glog.Infof("public server: closing") @@ -149,6 +152,16 @@ func (s *PublicServer) OnNewTxAddr(txid string, addr string, isOutput bool) { s.socketio.OnNewTxAddr(txid, addr, isOutput) } +func (s *PublicServer) txRedirect(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) + s.metrics.ExplorerViews.With(common.Labels{"action": "tx-redirect"}).Inc() +} + +func (s *PublicServer) addressRedirect(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) + s.metrics.ExplorerViews.With(common.Labels{"action": "address-redirect"}).Inc() +} + func splitBinding(binding string) (addr string, path string) { i := strings.Index(binding, "/") if i >= 0 { @@ -167,135 +180,397 @@ func joinURL(base string, part string) string { return part } -func (s *PublicServer) txRedirect(w http.ResponseWriter, r *http.Request) { - if s.explorerURL != "" { - http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) - s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() +func getFunctionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} + +func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, error)) func(w http.ResponseWriter, r *http.Request) { + type jsonError struct { + Error string `json:"error"` + } + return func(w http.ResponseWriter, r *http.Request) { + var data interface{} + var err error + defer func() { + if e := recover(); e != nil { + glog.Error(getFunctionName(handler), " recovered from panic: ", e) + if s.debug { + data = jsonError{fmt.Sprint("Internal server error: recovered from panic ", e)} + } else { + data = jsonError{"Internal server error"} + } + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + if _, isError := data.(jsonError); isError { + w.WriteHeader(http.StatusInternalServerError) + } + json.NewEncoder(w).Encode(data) + }() + data, err = handler(r) + if err != nil || data == nil { + if apiErr, ok := err.(*api.ApiError); ok { + data = jsonError{apiErr.Error()} + } else { + if err != nil { + glog.Error(getFunctionName(handler), " error: ", err) + } + if s.debug { + data = jsonError{fmt.Sprintf("Internal server error: %v, data %+v", err, data)} + } else { + data = jsonError{"Internal server error"} + } + } + } } } -func (s *PublicServer) addressRedirect(w http.ResponseWriter, r *http.Request) { - if s.explorerURL != "" { - http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) - s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() +func (s *PublicServer) newTemplateData() *TemplateData { + return &TemplateData{ + CoinName: s.is.Coin, + CoinShortcut: s.is.CoinShortcut, + InternalExplorer: s.internalExplorer && !s.is.InitialSync, } } +func (s *PublicServer) newTemplateDataWithError(text string) *TemplateData { + td := s.newTemplateData() + td.Error = &api.ApiError{Text: text} + return td +} + +func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + var t tpl + var data *TemplateData + var err error + defer func() { + if e := recover(); e != nil { + glog.Error(getFunctionName(handler), " recovered from panic: ", e) + t = errorInternalTpl + if s.debug { + data = s.newTemplateDataWithError(fmt.Sprint("Internal server error: recovered from panic ", e)) + } else { + data = s.newTemplateDataWithError("Internal server error") + } + } + // noTpl means the handler completely handled the request + if t != noTpl { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + // return 500 Internal Server Error with errorInternalTpl + if t == errorInternalTpl { + w.WriteHeader(http.StatusInternalServerError) + } + if err := s.templates[t].ExecuteTemplate(w, "base.html", data); err != nil { + glog.Error(err) + } + } + }() + if s.debug { + // reload templates on each request + // to reflect changes during development + s.templates = parseTemplates() + } + t, data, err = handler(w, r) + if err != nil || (data == nil && t != noTpl) { + t = errorInternalTpl + if apiErr, ok := err.(*api.ApiError); ok { + data = s.newTemplateData() + data.Error = apiErr + if apiErr.Public { + t = errorTpl + } + } else { + if err != nil { + glog.Error(getFunctionName(handler), " error: ", err) + } + if s.debug { + data = s.newTemplateDataWithError(fmt.Sprintf("Internal server error: %v, data %+v", err, data)) + } else { + data = s.newTemplateDataWithError("Internal server error") + } + } + } + } +} + +type tpl int + +const ( + noTpl = tpl(iota) + errorTpl + errorInternalTpl + indexTpl + txTpl + addressTpl + blocksTpl + blockTpl + + tplCount +) + type TemplateData struct { - CoinName string - CoinShortcut string - Address *api.Address - AddrStr string - Tx *api.Tx + CoinName string + CoinShortcut string + InternalExplorer bool + Address *api.Address + AddrStr string + Tx *api.Tx + Error *api.ApiError + Blocks *api.Blocks + Block *api.Block + Info *api.SystemInfo + Page int + PrevPage int + NextPage int + PagingRange []int } +func parseTemplates() []*template.Template { + templateFuncMap := template.FuncMap{ + "formatTime": formatTime, + "formatUnixTime": formatUnixTime, + "formatAmount": formatAmount, + "setTxToTemplateData": setTxToTemplateData, + "stringInSlice": stringInSlice, + } + t := make([]*template.Template, tplCount) + t[errorTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) + t[errorInternalTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) + t[indexTpl] = template.Must(template.New("index").Funcs(templateFuncMap).ParseFiles("./static/templates/index.html", "./static/templates/base.html")) + t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) + t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) + t[blocksTpl] = template.Must(template.New("blocks").Funcs(templateFuncMap).ParseFiles("./static/templates/blocks.html", "./static/templates/paging.html", "./static/templates/base.html")) + t[blockTpl] = template.Must(template.New("block").Funcs(templateFuncMap).ParseFiles("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) + return t +} + +func formatUnixTime(ut int64) string { + return formatTime(time.Unix(ut, 0)) +} + +func formatTime(t time.Time) string { + return t.Format(time.RFC1123) +} + +// for now return the string as it is +// in future could be used to do coin specific formatting +func formatAmount(a string) string { + return a +} + +// called from template to support txdetail.html functionality func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { td.Tx = tx return td } -func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx + s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] bestheight, _, err := s.db.GetBestBlock() if err == nil { - tx, err = s.api.GetTransaction(txid, bestheight, true) + tx, err = s.api.GetTransaction(txid, bestheight, false) } if err != nil { - glog.Error(err) + return errorTpl, nil, err } } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - - // temporarily reread the template on each request - // to reflect changes during development - s.txTpl, s.addressTpl = parseTemplates() - - data := &TemplateData{ - CoinName: s.is.Coin, - CoinShortcut: s.is.CoinShortcut, - Tx: tx, - } - if err := s.txTpl.ExecuteTemplate(w, "base.html", data); err != nil { - glog.Error(err) - } + data := s.newTemplateData() + data.Tx = tx + return txTpl, data, nil } -func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) explorerSpendingTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + s.metrics.ExplorerViews.With(common.Labels{"action": "spendingtx"}).Inc() + var err error + parts := strings.Split(r.URL.Path, "/") + if len(parts) > 2 { + tx := parts[len(parts)-2] + n, ec := strconv.Atoi(parts[len(parts)-1]) + if ec == nil { + spendingTx, err := s.api.GetSpendingTxid(tx, n) + if err == nil && spendingTx != "" { + http.Redirect(w, r, joinURL("/tx/", spendingTx), 302) + return noTpl, nil, nil + } + } + } + if err == nil { + err = api.NewApiError("Transaction not found", true) + } + return errorTpl, nil, err +} + +func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var address *api.Address var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { page = 0 } - addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsOnPage, false) if err != nil { - glog.Error(err) + return errorTpl, nil, err } } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - - // temporarily reread the template on each request - // to reflect changes during development - s.txTpl, s.addressTpl = parseTemplates() - - data := &TemplateData{ - CoinName: s.is.Coin, - CoinShortcut: s.is.CoinShortcut, - AddrStr: address.AddrStr, - Address: address, - } - if err := s.addressTpl.ExecuteTemplate(w, "base.html", data); err != nil { - glog.Error(err) - } + data := s.newTemplateData() + data.AddrStr = address.AddrStr + data.Address = address + data.Page = address.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + return addressTpl, data, nil } -type resAboutBlockbookPublic struct { - Coin string `json:"coin"` - Host string `json:"host"` - Version string `json:"version"` - GitCommit string `json:"gitcommit"` - BuildTime string `json:"buildtime"` - InSync bool `json:"inSync"` - BestHeight uint32 `json:"bestHeight"` - LastBlockTime time.Time `json:"lastBlockTime"` - InSyncMempool bool `json:"inSyncMempool"` - LastMempoolTime time.Time `json:"lastMempoolTime"` - About string `json:"about"` -} - -func (s *PublicServer) index(w http.ResponseWriter, r *http.Request) { - vi := common.GetVersionInfo() - ss, bh, st := s.is.GetSyncState() - ms, mt, _ := s.is.GetMempoolSyncState() - a := resAboutBlockbookPublic{ - Coin: s.is.Coin, - Host: s.is.Host, - Version: vi.Version, - GitCommit: vi.GitCommit, - BuildTime: vi.BuildTime, - InSync: ss, - BestHeight: bh, - LastBlockTime: st, - InSyncMempool: ms, - LastMempoolTime: mt, - About: blockbookAbout, +func (s *PublicServer) explorerBlocks(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var blocks *api.Blocks + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "blocks"}).Inc() + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - buf, err := json.MarshalIndent(a, "", " ") + blocks, err = s.api.GetBlocks(page, blocksOnPage) if err != nil { - glog.Error(err) + return errorTpl, nil, err } - w.Write(buf) + data := s.newTemplateData() + data.Blocks = blocks + data.Page = blocks.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(blocks.Page, blocks.TotalPages) + return blocksTpl, data, nil } -func (s *PublicServer) apiBlockIndex(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) explorerBlock(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var block *api.Block + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "block"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + block, err = s.api.GetBlock(r.URL.Path[i+1:], page, txsOnPage) + if err != nil { + return errorTpl, nil, err + } + } + data := s.newTemplateData() + data.Block = block + data.Page = block.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(block.Page, block.TotalPages) + return blockTpl, data, nil +} + +func (s *PublicServer) explorerIndex(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var si *api.SystemInfo + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "index"}).Inc() + si, err = s.api.GetSystemInfo(false) + if err != nil { + return errorTpl, nil, err + } + data := s.newTemplateData() + data.Info = si + return indexTpl, data, nil +} + +func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + q := strings.TrimSpace(r.URL.Query().Get("q")) + var tx *api.Tx + var address *api.Address + var block *api.Block + var bestheight uint32 + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() + if len(q) > 0 { + block, err = s.api.GetBlock(q, 0, 1) + if err == nil { + http.Redirect(w, r, joinURL("/block/", block.Hash), 302) + return noTpl, nil, nil + } + bestheight, _, err = s.db.GetBestBlock() + if err == nil { + tx, err = s.api.GetTransaction(q, bestheight, false) + if err == nil { + http.Redirect(w, r, joinURL("/tx/", tx.Txid), 302) + return noTpl, nil, nil + } + } + address, err = s.api.GetAddress(q, 0, 1, true) + if err == nil { + http.Redirect(w, r, joinURL("/address/", address.AddrStr), 302) + return noTpl, nil, nil + } + } + return errorTpl, nil, api.NewApiError(fmt.Sprintf("No matching records found for '%v'", q), true) +} + +func getPagingRange(page int, total int) ([]int, int, int) { + if total < 2 { + return nil, 0, 0 + } + pp, np := page-1, page+1 + if np > total { + np = total + } + if pp < 1 { + pp = 1 + } + r := make([]int, 0, 8) + if total < 6 { + for i := 1; i <= total; i++ { + r = append(r, i) + } + } else { + r = append(r, 1) + if page > 3 { + r = append(r, 0) + } + if pp == 1 { + if page == 1 { + r = append(r, np) + r = append(r, np+1) + r = append(r, np+2) + } else { + r = append(r, page) + r = append(r, np) + r = append(r, np+1) + } + } else if np == total { + if page == total { + r = append(r, pp-2) + r = append(r, pp-1) + r = append(r, pp) + } else { + r = append(r, pp-1) + r = append(r, pp) + r = append(r, page) + } + } else { + r = append(r, pp) + r = append(r, page) + r = append(r, np) + } + if page <= total-3 { + r = append(r, 0) + } + r = append(r, total) + } + return r, pp, np +} + +func (s *PublicServer) apiIndex(r *http.Request) (interface{}, error) { + s.metrics.ExplorerViews.With(common.Labels{"action": "api-index"}).Inc() + return s.api.GetSystemInfo(false) +} + +func (s *PublicServer) apiBlockIndex(r *http.Request) (interface{}, error) { type resBlockIndex struct { BlockHash string `json:"blockHash"` - About string `json:"about"` } var err error var hash string @@ -312,50 +587,51 @@ func (s *PublicServer) apiBlockIndex(w http.ResponseWriter, r *http.Request) { } if err != nil { glog.Error(err) - } else { - r := resBlockIndex{ - BlockHash: hash, - About: blockbookAbout, - } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(r) + return nil, err } + return resBlockIndex{ + BlockHash: hash, + }, nil } -func (s *PublicServer) apiTx(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) { var tx *api.Tx var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-tx"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] bestheight, _, err := s.db.GetBestBlock() if err == nil { tx, err = s.api.GetTransaction(txid, bestheight, true) - } else { - glog.Error(err) } } - if err == nil { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(tx) - } + return tx, err } -func (s *PublicServer) apiAddress(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { var address *api.Address var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-address"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { page = 0 } - addrID := r.URL.Path[i+1:] - address, err = s.api.GetAddress(addrID, page) - if err != nil { - glog.Error(err) - } - } - if err == nil { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - json.NewEncoder(w).Encode(address) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsInAPI, true) } + return address, err +} + +func (s *PublicServer) apiBlock(r *http.Request) (interface{}, error) { + var block *api.Block + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-block"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + block, err = s.api.GetBlock(r.URL.Path[i+1:], page, txsInAPI) + } + return block, err } diff --git a/server/socketio.go b/server/socketio.go index 241df197..2e931d1f 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -7,6 +7,7 @@ import ( "blockbook/db" "encoding/json" "net/http" + "strconv" "strings" "time" @@ -305,6 +306,29 @@ func txToResTx(tx *bchain.Tx, height int, hi []txInputs, ho []txOutputs) resTx { } } +func addressInSlice(s, t []string) string { + for _, sa := range s { + for _, ta := range t { + if ta == sa { + return sa + } + } + } + return "" +} + +func (s *SocketIoServer) getAddressesFromVout(vout *bchain.Vout) ([]string, error) { + addrDesc, err := s.chainParser.GetAddrDescFromVout(vout) + if err != nil { + return nil, err + } + voutAddr, _, err := s.chainParser.GetAddressesFromAddrDesc(addrDesc) + if err != nil { + return nil, err + } + return voutAddr, nil +} + func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res resultGetAddressHistory, err error) { txr, err := s.getAddressTxids(addr, opts) if err != nil { @@ -329,22 +353,26 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r for _, vout := range tx.Vout { aoh := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.Int64(), Script: &aoh, } - if vout.Address != nil { - a := vout.Address.String() - ao.Address = &a - if vout.Address.InSlice(addr) { - hi, ok := ads[a] - if ok { - hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) - } else { - hi := addressHistoryIndexes{} - hi.InputIndexes = make([]int, 0) - hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) - ads[a] = hi - } + voutAddr, err := s.getAddressesFromVout(&vout) + if err != nil { + return res, err + } + if len(voutAddr) > 0 { + ao.Address = &voutAddr[0] + } + a := addressInSlice(voutAddr, addr) + if a != "" { + hi, ok := ads[a] + if ok { + hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) + } else { + hi := addressHistoryIndexes{} + hi.InputIndexes = make([]int, 0) + hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) + ads[a] = hi } } ho = append(ho, ao) @@ -452,6 +480,7 @@ func unmarshalEstimateSmartFee(params []byte) (blocks int, conservative bool, er } type resultEstimateSmartFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -460,7 +489,7 @@ func (s *SocketIoServer) estimateSmartFee(blocks int, conservative bool) (res re if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -479,6 +508,7 @@ func unmarshalEstimateFee(params []byte) (blocks int, err error) { } type resultEstimateFee struct { + // for compatibility reasons use float64 Result float64 `json:"result"` } @@ -487,7 +517,7 @@ func (s *SocketIoServer) estimateFee(blocks int) (res resultEstimateFee, err err if err != nil { return } - res.Result = fee + res.Result, err = strconv.ParseFloat(s.chainParser.AmountToDecimalString(&fee), 64) return } @@ -521,7 +551,7 @@ func (s *SocketIoServer) getInfo() (res resultGetInfo, err error) { res.Result.Network = s.chain.GetNetworkName() res.Result.Subversion = s.chain.GetSubversion() res.Result.CoinName = s.chain.GetCoinName() - res.Result.About = blockbookAbout + res.Result.About = api.BlockbookAbout return } @@ -578,17 +608,38 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai OutputIndex: int(vin.Vout), } if vin.Txid != "" { - otx, _, err := s.txCache.GetTransaction(vin.Txid, bestheight) + var voutAddr []string + // load spending addresses from TxAddresses + ta, err := s.db.GetTxAddresses(vin.Txid) if err != nil { return res, err } - if len(otx.Vout) > int(vin.Vout) { - vout := &otx.Vout[vin.Vout] - if vout.Address != nil { - a := vout.Address.String() - ai.Address = &a + if ta == nil { + // the tx may be in mempool, try to load it from backend + otx, _, err := s.txCache.GetTransaction(vin.Txid, bestheight) + if err != nil { + return res, err } - ai.Satoshis = int64(vout.Value*1E8 + 0.5) + if len(otx.Vout) > int(vin.Vout) { + vout := &otx.Vout[vin.Vout] + voutAddr, err = s.getAddressesFromVout(vout) + if err != nil { + return res, err + } + ai.Satoshis = vout.ValueSat.Int64() + } + } else { + if len(ta.Outputs) > int(vin.Vout) { + output := &ta.Outputs[vin.Vout] + ai.Satoshis = output.ValueSat.Int64() + voutAddr, _, err = output.Addresses(s.chainParser) + if err != nil { + return res, err + } + } + } + if len(voutAddr) > 0 { + ai.Address = &voutAddr[0] } } hi = append(hi, ai) @@ -596,12 +647,15 @@ func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetai for _, vout := range tx.Vout { aos := vout.ScriptPubKey.Hex ao := txOutputs{ - Satoshis: int64(vout.Value*1E8 + 0.5), + Satoshis: vout.ValueSat.Int64(), Script: &aos, } - if vout.Address != nil { - a := vout.Address.String() - ao.Address = &a + voutAddr, err := s.getAddressesFromVout(&vout) + if err != nil { + return res, err + } + if len(voutAddr) > 0 { + ao.Address = &voutAddr[0] } ho = append(ho, ao) } diff --git a/server/socketio_test.go b/server/socketio_test.go index 108815cd..09f2d368 100644 --- a/server/socketio_test.go +++ b/server/socketio_test.go @@ -1,4 +1,4 @@ -// +build unittest +// +build integration package server diff --git a/static/css/main.css b/static/css/main.css index 8265aadd..b18df624 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -15,6 +15,14 @@ a { text-decoration: none; } +h1 small { + font-size: 65%; +} + +h3 { + margin-top: 20px; +} + .octicon { color: #777; display: inline-block; @@ -23,6 +31,22 @@ a { height: 16px; } +.navbar-form { + padding-bottom: 1px; +} + +.navbar-form .form-control { + background-color: gray; + color: #fff; + border-radius: 3px; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border: 0; + -webkit-box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); + -moz-box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); + box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); +} + @media (min-width: 768px) { .container { max-width: 750px; @@ -39,6 +63,9 @@ a { .octicon { height: 24px; } + .navbar-form .form-control { + width: 230px; + } } @media (min-width: 1200px) { @@ -51,6 +78,9 @@ a { .octicon { height: 32px; } + .navbar-form .form-control { + width: 360px; + } } #header { @@ -75,7 +105,6 @@ a { color: rgba(255, 255, 255); font-weight: bold; font-size: 19px; - padding-left: 35px; } .trezor-logo-svg-white svg { @@ -106,7 +135,7 @@ a { .line-top { border-top: 1px solid #EAEAEA; - padding: 15px 0 0; + padding: 10px 0 0; } .line-mid { @@ -128,6 +157,11 @@ a { white-space: nowrap; vertical-align: baseline; border-radius: .25em; + margin-top: 5px; +} + +.txvalues:not(:last-child) { + margin-right: 5px; } .txvalues-default { @@ -149,14 +183,17 @@ a { } .ellipsis { - display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .data-div { - margin: 20px 0; + margin: 20px 0 30px 0; +} + +.data-div .col-md-10 { + padding-left: 0; } .data-table { @@ -169,10 +206,69 @@ a { padding: .4rem; } +.data-table span.ellipsis { + max-width: 100%; +} + .data { font-weight: bold; } .alert .data-table { margin: 0; +} + +.navbar-nav .nav-link { + padding-right: 0; + padding-left: .25rem; +} + +.navbar-text .nav-link { + padding: 0; +} + +::-webkit-input-placeholder { + color: #CCC!important; + font-style: italic; + font-size: 14px; +} + +::-moz-placeholder { + color: #CCC!important; + font-style: italic; + font-size: 14px; +} + +.h-container { + display: -webkit-box; + display: -ms-flexbox; + -ms-flex-wrap: wrap; + flex-wrap: wrap; +} + +.h-container-6 { + width: 50%; + margin: 0; +} + +.h-container ul { + margin: 0; +} + +.page-link { + color: #428bca; +} + +.page-text { + display: block; + padding: .5rem .3rem; + line-height: 1.25; +} + +.page-link { + color: #428bca; +} + +.page-item.active .page-link { + background-color: #428bca; } \ No newline at end of file diff --git a/static/js/qrcode.min.js b/static/js/qrcode.min.js new file mode 100755 index 00000000..993e88f3 --- /dev/null +++ b/static/js/qrcode.min.js @@ -0,0 +1 @@ +var QRCode;!function(){function a(a){this.mode=c.MODE_8BIT_BYTE,this.data=a,this.parsedData=[];for(var b=[],d=0,e=this.data.length;e>d;d++){var f=this.data.charCodeAt(d);f>65536?(b[0]=240|(1835008&f)>>>18,b[1]=128|(258048&f)>>>12,b[2]=128|(4032&f)>>>6,b[3]=128|63&f):f>2048?(b[0]=224|(61440&f)>>>12,b[1]=128|(4032&f)>>>6,b[2]=128|63&f):f>128?(b[0]=192|(1984&f)>>>6,b[1]=128|63&f):b[0]=f,this.parsedData=this.parsedData.concat(b)}this.parsedData.length!=this.data.length&&(this.parsedData.unshift(191),this.parsedData.unshift(187),this.parsedData.unshift(239))}function b(a,b){this.typeNumber=a,this.errorCorrectLevel=b,this.modules=null,this.moduleCount=0,this.dataCache=null,this.dataList=[]}function i(a,b){if(void 0==a.length)throw new Error(a.length+"/"+b);for(var c=0;c=f;f++){var h=0;switch(b){case d.L:h=l[f][0];break;case d.M:h=l[f][1];break;case d.Q:h=l[f][2];break;case d.H:h=l[f][3]}if(h>=e)break;c++}if(c>l.length)throw new Error("Too long data");return c}function s(a){var b=encodeURI(a).toString().replace(/\%[0-9a-fA-F]{2}/g,"a");return b.length+(b.length!=a?3:0)}a.prototype={getLength:function(){return this.parsedData.length},write:function(a){for(var b=0,c=this.parsedData.length;c>b;b++)a.put(this.parsedData[b],8)}},b.prototype={addData:function(b){var c=new a(b);this.dataList.push(c),this.dataCache=null},isDark:function(a,b){if(0>a||this.moduleCount<=a||0>b||this.moduleCount<=b)throw new Error(a+","+b);return this.modules[a][b]},getModuleCount:function(){return this.moduleCount},make:function(){this.makeImpl(!1,this.getBestMaskPattern())},makeImpl:function(a,c){this.moduleCount=4*this.typeNumber+17,this.modules=new Array(this.moduleCount);for(var d=0;d=7&&this.setupTypeNumber(a),null==this.dataCache&&(this.dataCache=b.createData(this.typeNumber,this.errorCorrectLevel,this.dataList)),this.mapData(this.dataCache,c)},setupPositionProbePattern:function(a,b){for(var c=-1;7>=c;c++)if(!(-1>=a+c||this.moduleCount<=a+c))for(var d=-1;7>=d;d++)-1>=b+d||this.moduleCount<=b+d||(this.modules[a+c][b+d]=c>=0&&6>=c&&(0==d||6==d)||d>=0&&6>=d&&(0==c||6==c)||c>=2&&4>=c&&d>=2&&4>=d?!0:!1)},getBestMaskPattern:function(){for(var a=0,b=0,c=0;8>c;c++){this.makeImpl(!0,c);var d=f.getLostPoint(this);(0==c||a>d)&&(a=d,b=c)}return b},createMovieClip:function(a,b,c){var d=a.createEmptyMovieClip(b,c),e=1;this.make();for(var f=0;f=g;g++)for(var h=-2;2>=h;h++)this.modules[d+g][e+h]=-2==g||2==g||-2==h||2==h||0==g&&0==h?!0:!1}},setupTypeNumber:function(a){for(var b=f.getBCHTypeNumber(this.typeNumber),c=0;18>c;c++){var d=!a&&1==(1&b>>c);this.modules[Math.floor(c/3)][c%3+this.moduleCount-8-3]=d}for(var c=0;18>c;c++){var d=!a&&1==(1&b>>c);this.modules[c%3+this.moduleCount-8-3][Math.floor(c/3)]=d}},setupTypeInfo:function(a,b){for(var c=this.errorCorrectLevel<<3|b,d=f.getBCHTypeInfo(c),e=0;15>e;e++){var g=!a&&1==(1&d>>e);6>e?this.modules[e][8]=g:8>e?this.modules[e+1][8]=g:this.modules[this.moduleCount-15+e][8]=g}for(var e=0;15>e;e++){var g=!a&&1==(1&d>>e);8>e?this.modules[8][this.moduleCount-e-1]=g:9>e?this.modules[8][15-e-1+1]=g:this.modules[8][15-e-1]=g}this.modules[this.moduleCount-8][8]=!a},mapData:function(a,b){for(var c=-1,d=this.moduleCount-1,e=7,g=0,h=this.moduleCount-1;h>0;h-=2)for(6==h&&h--;;){for(var i=0;2>i;i++)if(null==this.modules[d][h-i]){var j=!1;g>>e));var k=f.getMask(b,d,h-i);k&&(j=!j),this.modules[d][h-i]=j,e--,-1==e&&(g++,e=7)}if(d+=c,0>d||this.moduleCount<=d){d-=c,c=-c;break}}}},b.PAD0=236,b.PAD1=17,b.createData=function(a,c,d){for(var e=j.getRSBlocks(a,c),g=new k,h=0;h8*l)throw new Error("code length overflow. ("+g.getLengthInBits()+">"+8*l+")");for(g.getLengthInBits()+4<=8*l&&g.put(0,4);0!=g.getLengthInBits()%8;)g.putBit(!1);for(;;){if(g.getLengthInBits()>=8*l)break;if(g.put(b.PAD0,8),g.getLengthInBits()>=8*l)break;g.put(b.PAD1,8)}return b.createBytes(g,e)},b.createBytes=function(a,b){for(var c=0,d=0,e=0,g=new Array(b.length),h=new Array(b.length),j=0;j=0?p.get(q):0}}for(var r=0,m=0;mm;m++)for(var j=0;jm;m++)for(var j=0;j=0;)b^=f.G15<=0;)b^=f.G18<>>=1;return b},getPatternPosition:function(a){return f.PATTERN_POSITION_TABLE[a-1]},getMask:function(a,b,c){switch(a){case e.PATTERN000:return 0==(b+c)%2;case e.PATTERN001:return 0==b%2;case e.PATTERN010:return 0==c%3;case e.PATTERN011:return 0==(b+c)%3;case e.PATTERN100:return 0==(Math.floor(b/2)+Math.floor(c/3))%2;case e.PATTERN101:return 0==b*c%2+b*c%3;case e.PATTERN110:return 0==(b*c%2+b*c%3)%2;case e.PATTERN111:return 0==(b*c%3+(b+c)%2)%2;default:throw new Error("bad maskPattern:"+a)}},getErrorCorrectPolynomial:function(a){for(var b=new i([1],0),c=0;a>c;c++)b=b.multiply(new i([1,g.gexp(c)],0));return b},getLengthInBits:function(a,b){if(b>=1&&10>b)switch(a){case c.MODE_NUMBER:return 10;case c.MODE_ALPHA_NUM:return 9;case c.MODE_8BIT_BYTE:return 8;case c.MODE_KANJI:return 8;default:throw new Error("mode:"+a)}else if(27>b)switch(a){case c.MODE_NUMBER:return 12;case c.MODE_ALPHA_NUM:return 11;case c.MODE_8BIT_BYTE:return 16;case c.MODE_KANJI:return 10;default:throw new Error("mode:"+a)}else{if(!(41>b))throw new Error("type:"+b);switch(a){case c.MODE_NUMBER:return 14;case c.MODE_ALPHA_NUM:return 13;case c.MODE_8BIT_BYTE:return 16;case c.MODE_KANJI:return 12;default:throw new Error("mode:"+a)}}},getLostPoint:function(a){for(var b=a.getModuleCount(),c=0,d=0;b>d;d++)for(var e=0;b>e;e++){for(var f=0,g=a.isDark(d,e),h=-1;1>=h;h++)if(!(0>d+h||d+h>=b))for(var i=-1;1>=i;i++)0>e+i||e+i>=b||(0!=h||0!=i)&&g==a.isDark(d+h,e+i)&&f++;f>5&&(c+=3+f-5)}for(var d=0;b-1>d;d++)for(var e=0;b-1>e;e++){var j=0;a.isDark(d,e)&&j++,a.isDark(d+1,e)&&j++,a.isDark(d,e+1)&&j++,a.isDark(d+1,e+1)&&j++,(0==j||4==j)&&(c+=3)}for(var d=0;b>d;d++)for(var e=0;b-6>e;e++)a.isDark(d,e)&&!a.isDark(d,e+1)&&a.isDark(d,e+2)&&a.isDark(d,e+3)&&a.isDark(d,e+4)&&!a.isDark(d,e+5)&&a.isDark(d,e+6)&&(c+=40);for(var e=0;b>e;e++)for(var d=0;b-6>d;d++)a.isDark(d,e)&&!a.isDark(d+1,e)&&a.isDark(d+2,e)&&a.isDark(d+3,e)&&a.isDark(d+4,e)&&!a.isDark(d+5,e)&&a.isDark(d+6,e)&&(c+=40);for(var k=0,e=0;b>e;e++)for(var d=0;b>d;d++)a.isDark(d,e)&&k++;var l=Math.abs(100*k/b/b-50)/5;return c+=10*l}},g={glog:function(a){if(1>a)throw new Error("glog("+a+")");return g.LOG_TABLE[a]},gexp:function(a){for(;0>a;)a+=255;for(;a>=256;)a-=255;return g.EXP_TABLE[a]},EXP_TABLE:new Array(256),LOG_TABLE:new Array(256)},h=0;8>h;h++)g.EXP_TABLE[h]=1<h;h++)g.EXP_TABLE[h]=g.EXP_TABLE[h-4]^g.EXP_TABLE[h-5]^g.EXP_TABLE[h-6]^g.EXP_TABLE[h-8];for(var h=0;255>h;h++)g.LOG_TABLE[g.EXP_TABLE[h]]=h;i.prototype={get:function(a){return this.num[a]},getLength:function(){return this.num.length},multiply:function(a){for(var b=new Array(this.getLength()+a.getLength()-1),c=0;cf;f++)for(var g=c[3*f+0],h=c[3*f+1],i=c[3*f+2],k=0;g>k;k++)e.push(new j(h,i));return e},j.getRsBlockTable=function(a,b){switch(b){case d.L:return j.RS_BLOCK_TABLE[4*(a-1)+0];case d.M:return j.RS_BLOCK_TABLE[4*(a-1)+1];case d.Q:return j.RS_BLOCK_TABLE[4*(a-1)+2];case d.H:return j.RS_BLOCK_TABLE[4*(a-1)+3];default:return void 0}},k.prototype={get:function(a){var b=Math.floor(a/8);return 1==(1&this.buffer[b]>>>7-a%8)},put:function(a,b){for(var c=0;b>c;c++)this.putBit(1==(1&a>>>b-c-1))},getLengthInBits:function(){return this.length},putBit:function(a){var b=Math.floor(this.length/8);this.buffer.length<=b&&this.buffer.push(0),a&&(this.buffer[b]|=128>>>this.length%8),this.length++}};var l=[[17,14,11,7],[32,26,20,14],[53,42,32,24],[78,62,46,34],[106,84,60,44],[134,106,74,58],[154,122,86,64],[192,152,108,84],[230,180,130,98],[271,213,151,119],[321,251,177,137],[367,287,203,155],[425,331,241,177],[458,362,258,194],[520,412,292,220],[586,450,322,250],[644,504,364,280],[718,560,394,310],[792,624,442,338],[858,666,482,382],[929,711,509,403],[1003,779,565,439],[1091,857,611,461],[1171,911,661,511],[1273,997,715,535],[1367,1059,751,593],[1465,1125,805,625],[1528,1190,868,658],[1628,1264,908,698],[1732,1370,982,742],[1840,1452,1030,790],[1952,1538,1112,842],[2068,1628,1168,898],[2188,1722,1228,958],[2303,1809,1283,983],[2431,1911,1351,1051],[2563,1989,1423,1093],[2699,2099,1499,1139],[2809,2213,1579,1219],[2953,2331,1663,1273]],o=function(){var a=function(a,b){this._el=a,this._htOption=b};return a.prototype.draw=function(a){function g(a,b){var c=document.createElementNS("http://www.w3.org/2000/svg",a);for(var d in b)b.hasOwnProperty(d)&&c.setAttribute(d,b[d]);return c}var b=this._htOption,c=this._el,d=a.getModuleCount();Math.floor(b.width/d),Math.floor(b.height/d),this.clear();var h=g("svg",{viewBox:"0 0 "+String(d)+" "+String(d),width:"100%",height:"100%",fill:b.colorLight});h.setAttributeNS("http://www.w3.org/2000/xmlns/","xmlns:xlink","http://www.w3.org/1999/xlink"),c.appendChild(h),h.appendChild(g("rect",{fill:b.colorDark,width:"1",height:"1",id:"template"}));for(var i=0;d>i;i++)for(var j=0;d>j;j++)if(a.isDark(i,j)){var k=g("use",{x:String(i),y:String(j)});k.setAttributeNS("http://www.w3.org/1999/xlink","href","#template"),h.appendChild(k)}},a.prototype.clear=function(){for(;this._el.hasChildNodes();)this._el.removeChild(this._el.lastChild)},a}(),p="svg"===document.documentElement.tagName.toLowerCase(),q=p?o:m()?function(){function a(){this._elImage.src=this._elCanvas.toDataURL("image/png"),this._elImage.style.display="block",this._elCanvas.style.display="none"}function d(a,b){var c=this;if(c._fFail=b,c._fSuccess=a,null===c._bSupportDataURI){var d=document.createElement("img"),e=function(){c._bSupportDataURI=!1,c._fFail&&_fFail.call(c)},f=function(){c._bSupportDataURI=!0,c._fSuccess&&c._fSuccess.call(c)};return d.onabort=e,d.onerror=e,d.onload=f,d.src="data:image/gif;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==",void 0}c._bSupportDataURI===!0&&c._fSuccess?c._fSuccess.call(c):c._bSupportDataURI===!1&&c._fFail&&c._fFail.call(c)}if(this._android&&this._android<=2.1){var b=1/window.devicePixelRatio,c=CanvasRenderingContext2D.prototype.drawImage;CanvasRenderingContext2D.prototype.drawImage=function(a,d,e,f,g,h,i,j){if("nodeName"in a&&/img/i.test(a.nodeName))for(var l=arguments.length-1;l>=1;l--)arguments[l]=arguments[l]*b;else"undefined"==typeof j&&(arguments[1]*=b,arguments[2]*=b,arguments[3]*=b,arguments[4]*=b);c.apply(this,arguments)}}var e=function(a,b){this._bIsPainted=!1,this._android=n(),this._htOption=b,this._elCanvas=document.createElement("canvas"),this._elCanvas.width=b.width,this._elCanvas.height=b.height,a.appendChild(this._elCanvas),this._el=a,this._oContext=this._elCanvas.getContext("2d"),this._bIsPainted=!1,this._elImage=document.createElement("img"),this._elImage.style.display="none",this._el.appendChild(this._elImage),this._bSupportDataURI=null};return e.prototype.draw=function(a){var b=this._elImage,c=this._oContext,d=this._htOption,e=a.getModuleCount(),f=d.width/e,g=d.height/e,h=Math.round(f),i=Math.round(g);b.style.display="none",this.clear();for(var j=0;e>j;j++)for(var k=0;e>k;k++){var l=a.isDark(j,k),m=k*f,n=j*g;c.strokeStyle=l?d.colorDark:d.colorLight,c.lineWidth=1,c.fillStyle=l?d.colorDark:d.colorLight,c.fillRect(m,n,f,g),c.strokeRect(Math.floor(m)+.5,Math.floor(n)+.5,h,i),c.strokeRect(Math.ceil(m)-.5,Math.ceil(n)-.5,h,i)}this._bIsPainted=!0},e.prototype.makeImage=function(){this._bIsPainted&&d.call(this,a)},e.prototype.isPainted=function(){return this._bIsPainted},e.prototype.clear=function(){this._oContext.clearRect(0,0,this._elCanvas.width,this._elCanvas.height),this._bIsPainted=!1},e.prototype.round=function(a){return a?Math.floor(1e3*a)/1e3:a},e}():function(){var a=function(a,b){this._el=a,this._htOption=b};return a.prototype.draw=function(a){for(var b=this._htOption,c=this._el,d=a.getModuleCount(),e=Math.floor(b.width/d),f=Math.floor(b.height/d),g=[''],h=0;d>h;h++){g.push("");for(var i=0;d>i;i++)g.push('');g.push("")}g.push("
"),c.innerHTML=g.join("");var j=c.childNodes[0],k=(b.width-j.offsetWidth)/2,l=(b.height-j.offsetHeight)/2;k>0&&l>0&&(j.style.margin=l+"px "+k+"px")},a.prototype.clear=function(){this._el.innerHTML=""},a}();QRCode=function(a,b){if(this._htOption={width:256,height:256,typeNumber:4,colorDark:"#000000",colorLight:"#ffffff",correctLevel:d.H},"string"==typeof b&&(b={text:b}),b)for(var c in b)this._htOption[c]=b[c];"string"==typeof a&&(a=document.getElementById(a)),this._android=n(),this._el=a,this._oQRCode=null,this._oDrawing=new q(this._el,this._htOption),this._htOption.text&&this.makeCode(this._htOption.text)},QRCode.prototype.makeCode=function(a){this._oQRCode=new b(r(a,this._htOption.correctLevel),this._htOption.correctLevel),this._oQRCode.addData(a),this._oQRCode.make(),this._el.title=a,this._oDrawing.draw(this._oQRCode),this.makeImage()},QRCode.prototype.makeImage=function(){"function"==typeof this._oDrawing.makeImage&&(!this._android||this._android>=3)&&this._oDrawing.makeImage()},QRCode.prototype.clear=function(){this._oDrawing.clear()},QRCode.CorrectLevel=d}(); \ No newline at end of file diff --git a/static/templates/address.html b/static/templates/address.html index 3a457703..7a56e729 100644 --- a/static/templates/address.html +++ b/static/templates/address.html @@ -2,33 +2,42 @@

Address {{formatAmount $addr.Balance}} {{$cs}}

-
- {{$addr.AddrStr}} +
+ {{$addr.AddrStr}}

Confirmed

-
- - - - - - - - - - - - - - - - - - - -
Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}
Final Balance{{formatAmount $addr.Balance}} {{$cs}}
No. Transactions{{$addr.TxApperances}}
+
+
+ + + + + + + + + + + + + + + + + + + +
Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}
Final Balance{{formatAmount $addr.Balance}} {{$cs}}
No. Transactions{{$addr.TxApperances}}
+
+
+
+ + +
-{{if $addr.UnconfirmedTxApperances}} +{{- if $addr.UnconfirmedTxApperances -}}

Unconfirmed

@@ -44,9 +53,13 @@
-{{end}} {{if $addr.Transactions}} -

Transactions

-
- {{range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data }}{{end}} +{{- end}}{{if $addr.Transactions -}} +
+

Transactions

+
-{{end}} {{end}} \ No newline at end of file +
+ {{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}} +
+ +{{end}}{{end}} \ No newline at end of file diff --git a/static/templates/base.html b/static/templates/base.html index 5ab2fb3e..05bebd41 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -14,9 +14,8 @@
- {{template "specific" .}} + {{- template "specific" . -}}