diff --git a/Gopkg.lock b/Gopkg.lock index eaa380d8..f9a70817 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -69,9 +69,9 @@ [[projects]] name = "github.com/ethereum/go-ethereum" - packages = [".","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","ethdb","log","metrics","params","rlp","rpc","trie"] - revision = "329ac18ef617d0238f71637bffe78f028b0f13f7" - version = "v1.8.3" + packages = [".","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","ethdb","log","metrics","p2p/netutil","params","rlp","rpc","trie"] + revision = "89451f7c382ad2185987ee369f16416f89c28a7d" + version = "v1.8.15" [[projects]] name = "github.com/go-stack/stack" diff --git a/api/types.go b/api/types.go index ed0a1a2d..32280427 100644 --- a/api/types.go +++ b/api/types.go @@ -2,9 +2,14 @@ package api import ( "blockbook/bchain" + "blockbook/common" + "blockbook/db" "math/big" + "time" ) +const BlockbookAbout = "Blockbook - blockchain indexer for TREZOR wallet https://trezor.io/. Do not use for any other purpose." + type ApiError struct { Text string Public bool @@ -73,10 +78,17 @@ type Tx struct { Size int `json:"size,omitempty"` ValueIn string `json:"valueIn"` Fees string `json:"fees"` - WithSpends bool `json:"withSpends,omitempty"` + Hex string `json:"hex"` +} + +type Paging struct { + Page int `json:"page"` + TotalPages int `json:"totalPages"` + ItemsOnPage int `json:"itemsOnPage"` } type Address struct { + Paging AddrStr string `json:"addrStr"` Balance string `json:"balance"` TotalReceived string `json:"totalReceived"` @@ -86,7 +98,41 @@ type Address struct { TxApperances int `json:"txApperances"` Transactions []*Tx `json:"txs,omitempty"` Txids []string `json:"transactions,omitempty"` - Page int `json:"page"` - TotalPages int `json:"totalPages"` - TxsOnPage int `json:"txsOnPage"` +} + +type Blocks struct { + Paging + Blocks []db.BlockInfo `json:"blocks"` +} + +type Block struct { + Paging + bchain.BlockInfo + TxCount int `json:"TxCount"` + Transactions []*Tx `json:"txs,omitempty"` +} + +type BlockbookInfo struct { + Coin string `json:"coin"` + Host string `json:"host"` + Version string `json:"version"` + GitCommit string `json:"gitcommit"` + BuildTime string `json:"buildtime"` + SyncMode bool `json:"syncMode"` + InitialSync bool `json:"initialsync"` + InSync bool `json:"inSync"` + BestHeight uint32 `json:"bestHeight"` + LastBlockTime time.Time `json:"lastBlockTime"` + InSyncMempool bool `json:"inSyncMempool"` + LastMempoolTime time.Time `json:"lastMempoolTime"` + MempoolSize int `json:"mempoolSize"` + DbSize int64 `json:"dbSize"` + DbSizeFromColumns int64 `json:"dbSizeFromColumns,omitempty"` + DbColumns []common.InternalStateColumn `json:"dbColumns,omitempty"` + About string `json:"about"` +} + +type SystemInfo struct { + Blockbook *BlockbookInfo `json:"blockbook"` + Backend *bchain.ChainInfo `json:"backend"` } diff --git a/api/worker.go b/api/worker.go index 7320134d..2dd6cb89 100644 --- a/api/worker.go +++ b/api/worker.go @@ -7,6 +7,7 @@ import ( "bytes" "fmt" "math/big" + "strconv" "time" "github.com/golang/glog" @@ -43,6 +44,61 @@ func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescript return addrDesc, a, s, err } +// setSpendingTxToVout is helper function, that finds transaction that spent given output and sets it to the output +// there is not an index, it must be found using addresses -> txaddresses -> tx +func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height, bestheight uint32) error { + err := w.db.GetAddrDescTransactions(vout.ScriptPubKey.AddrDesc, height, ^uint32(0), func(t string, index uint32, isOutput bool) error { + if isOutput == false { + tsp, err := w.db.GetTxAddresses(t) + if err != nil { + glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses") + } else { + if len(tsp.Inputs) > int(index) { + if tsp.Inputs[index].ValueSat.Cmp(&vout.ValueSat) == 0 { + spentTx, spentHeight, err := w.txCache.GetTransaction(t, bestheight) + if err != nil { + glog.Warning("Tx ", t, ": not found") + } else { + if len(spentTx.Vin) > int(index) { + if spentTx.Vin[index].Txid == txid { + vout.SpentTxID = t + vout.SpentHeight = int(spentHeight) + vout.SpentIndex = int(index) + return &db.StopIteration{} + } + } + } + } + } + } + } + return nil + }) + return err +} + +// GetSpendingTxid returns transaction id of transaction that spent given output +func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) { + start := time.Now() + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return "", err + } + tx, err := w.GetTransaction(txid, bestheight, false) + if err != nil { + return "", err + } + if n >= len(tx.Vout) || n < 0 { + return "", NewApiError(fmt.Sprintf("Passed incorrect vout index %v for tx %v, len vout %v", n, tx.Txid, len(tx.Vout)), false) + } + err = w.setSpendingTxToVout(&tx.Vout[n], tx.Txid, uint32(tx.Blockheight), bestheight) + if err != nil { + return "", err + } + glog.Info("GetSpendingTxid ", txid, " ", n, " finished in ", time.Since(start)) + return tx.Vout[n].SpentTxID, nil +} + // GetTransaction reads transaction data from txid func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool) (*Tx, error) { start := time.Now() @@ -124,37 +180,9 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool if ta != nil { vout.Spent = ta.Outputs[i].Spent if spendingTxs && vout.Spent { - // find transaction that spent this output - // there is not an index, it must be found in addresses -> txaddresses -> tx - err = w.db.GetAddrDescTransactions(vout.ScriptPubKey.AddrDesc, height, ^uint32(0), func(t string, index uint32, isOutput bool) error { - if isOutput == false { - tsp, err := w.db.GetTxAddresses(t) - if err != nil { - glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses") - } else { - if len(tsp.Inputs) > int(index) { - if tsp.Inputs[index].ValueSat.Cmp(&vout.ValueSat) == 0 { - spentTx, spentHeight, err := w.txCache.GetTransaction(t, bestheight) - if err != nil { - glog.Warning("Tx ", t, ": not found") - } else { - if len(spentTx.Vin) > int(index) { - if spentTx.Vin[index].Txid == bchainTx.Txid { - vout.SpentTxID = t - vout.SpentHeight = int(spentHeight) - vout.SpentIndex = int(index) - return &db.StopIteration{} - } - } - } - } - } - } - } - return nil - }) + err = w.setSpendingTxToVout(vout, bchainTx.Txid, height, bestheight) if err != nil { - glog.Errorf("GetAddrDescTransactions error %v, %v, output %v", err, vout.ScriptPubKey.AddrDesc) + glog.Errorf("setSpendingTxToVout error %v, %v, output %v", err, vout.ScriptPubKey.AddrDesc, vout.N) } } } @@ -173,12 +201,12 @@ func (w *Worker) GetTransaction(txid string, bestheight uint32, spendingTxs bool Confirmations: bchainTx.Confirmations, Fees: w.chainParser.AmountToDecimalString(&feesSat), Locktime: bchainTx.LockTime, - WithSpends: spendingTxs, Time: bchainTx.Time, Txid: bchainTx.Txid, ValueIn: w.chainParser.AmountToDecimalString(&valInSat), ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), Version: bchainTx.Version, + Hex: bchainTx.Hex, Vin: vins, Vout: vouts, } @@ -296,6 +324,27 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn return r } +func computePaging(count, page, itemsOnPage int) (Paging, int, int, int) { + from := page * itemsOnPage + totalPages := (count - 1) / itemsOnPage + if totalPages < 0 { + totalPages = 0 + } + if from >= count { + page = totalPages + } + from = page * itemsOnPage + to := (page + 1) * itemsOnPage + if to > count { + to = count + } + return Paging{ + ItemsOnPage: itemsOnPage, + Page: page + 1, + TotalPages: totalPages + 1, + }, from, to, page +} + // GetAddress computes address value and gets transactions for given address func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids bool) (*Address, error) { start := time.Now() @@ -341,23 +390,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b if err != nil { return nil, errors.Annotatef(err, "GetBestBlock") } - // paging - from := page * txsOnPage - totalPages := (len(txc) - 1) / txsOnPage - if totalPages < 0 { - totalPages = 0 - } - if from >= len(txc) { - page = totalPages - 1 - if page < 0 { - page = 0 - } - } - from = page * txsOnPage - to := (page + 1) * txsOnPage - if to > len(txc) { - to = len(txc) - } + pg, from, to, page := computePaging(len(txc), page, txsOnPage) var txs []*Tx var txids []string if onlyTxids { @@ -420,6 +453,7 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b txs = txs[:txi] } r := &Address{ + Paging: pg, AddrStr: address, Balance: w.chainParser.AmountToDecimalString(&ba.BalanceSat), TotalReceived: w.chainParser.AmountToDecimalString(ba.ReceivedSat()), @@ -429,10 +463,130 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b UnconfirmedTxApperances: len(txm), Transactions: txs, Txids: txids, - Page: page + 1, - TotalPages: totalPages + 1, - TxsOnPage: txsOnPage, } glog.Info("GetAddress ", address, " finished in ", time.Since(start)) return r, nil } + +// GetBlocks returns BlockInfo for blocks on given page +func (w *Worker) GetBlocks(page int, blocksOnPage int) (*Blocks, error) { + start := time.Now() + page-- + if page < 0 { + page = 0 + } + b, _, err := w.db.GetBestBlock() + bestheight := int(b) + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + pg, from, to, page := computePaging(bestheight+1, page, blocksOnPage) + r := &Blocks{Paging: pg} + r.Blocks = make([]db.BlockInfo, to-from) + for i := from; i < to; i++ { + bi, err := w.db.GetBlockInfo(uint32(bestheight - i)) + if err != nil { + return nil, err + } + r.Blocks[i-from] = *bi + } + glog.Info("GetBlocks page ", page, " finished in ", time.Since(start)) + return r, nil +} + +// GetBlock returns paged data about block +func (w *Worker) GetBlock(bid string, page int, txsOnPage int) (*Block, error) { + start := time.Now() + page-- + if page < 0 { + page = 0 + } + var hash string + height, err := strconv.Atoi(bid) + if err == nil && height < int(^uint32(0)) { + hash, err = w.db.GetBlockHash(uint32(height)) + } else { + hash = bid + } + bi, err := w.chain.GetBlockInfo(hash) + if err != nil { + if err == bchain.ErrBlockNotFound { + return nil, NewApiError("Block not found", true) + } + return nil, NewApiError(fmt.Sprintf("Block not found, %v", err), true) + } + dbi := &db.BlockInfo{ + Hash: bi.Hash, + Height: bi.Height, + Time: bi.Time, + } + txCount := len(bi.Txids) + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + pg, from, to, page := computePaging(txCount, page, txsOnPage) + glog.Info("GetBlock ", bid, ", page ", page, " finished in ", time.Since(start)) + txs := make([]*Tx, to-from) + txi := 0 + for i := from; i < to; i++ { + txid := bi.Txids[i] + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, dbi, bestheight) + txi++ + } + txs = txs[:txi] + bi.Txids = nil + return &Block{ + Paging: pg, + BlockInfo: *bi, + TxCount: txCount, + Transactions: txs, + }, nil +} + +// GetSystemInfo returns information about system +func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) { + start := time.Now() + ci, err := w.chain.GetChainInfo() + if err != nil { + return nil, errors.Annotatef(err, "GetChainInfo") + } + vi := common.GetVersionInfo() + ss, bh, st := w.is.GetSyncState() + ms, mt, msz := w.is.GetMempoolSyncState() + var dbc []common.InternalStateColumn + var dbs int64 + if internal { + dbc = w.is.GetAllDBColumnStats() + dbs = w.is.DBSizeTotal() + } + bi := &BlockbookInfo{ + Coin: w.is.Coin, + Host: w.is.Host, + Version: vi.Version, + GitCommit: vi.GitCommit, + BuildTime: vi.BuildTime, + SyncMode: w.is.SyncMode, + InitialSync: w.is.InitialSync, + InSync: ss, + BestHeight: bh, + LastBlockTime: st, + InSyncMempool: ms, + LastMempoolTime: mt, + MempoolSize: msz, + DbSize: w.db.DatabaseSizeOnDisk(), + DbSizeFromColumns: dbs, + DbColumns: dbc, + About: BlockbookAbout, + } + glog.Info("GetSystemInfo finished in ", time.Since(start)) + return &SystemInfo{bi, ci}, nil +} diff --git a/bchain/coins/bch/bcashparser_test.go b/bchain/coins/bch/bcashparser_test.go index d9e38ac7..71464b57 100644 --- a/bchain/coins/bch/bcashparser_test.go +++ b/bchain/coins/bch/bcashparser_test.go @@ -157,7 +157,7 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { { name: "OP_RETURN hex", parser: mainParserCashAddr, - addresses: []string{"OP_RETURN 07 2020f1686f6a20"}, + addresses: []string{"OP_RETURN 2020f1686f6a20"}, searchable: false, hex: "6a072020f1686f6a20", wantErr: false, diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index cbf4b94f..ce374430 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -101,7 +101,10 @@ func (b *BCashRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { if err != nil { return nil, errors.Annotatef(err, "hash %v", hash) } + // size is not returned by GetBlockHeader and would be overwritten + size := block.Size block.BlockHeader = *header + block.Size = size return block, nil } @@ -127,6 +130,28 @@ func (b *BCashRPC) GetBlockRaw(hash string) ([]byte, error) { return hex.DecodeString(res.Result) } +// GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids +func (b *BCashRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + glog.V(1).Info("rpc: getblock (verbosity=1) ", hash) + + res := btc.ResGetBlockInfo{} + req := cmdGetBlock{Method: "getblock"} + req.Params.BlockHash = hash + req.Params.Verbose = true + err := b.Call(&req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + if isErrBlockNotFound(res.Error) { + return nil, bchain.ErrBlockNotFound + } + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + return &res.Result, nil +} + // GetBlockFull returns block with given hash. func (b *BCashRPC) GetBlockFull(hash string) (*bchain.Block, error) { return nil, errors.New("Not implemented") diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 3e3f6ae0..35315d80 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -35,6 +35,7 @@ func init() { BlockChainFactories["Zcash"] = zec.NewZCashRPC BlockChainFactories["Zcash Testnet"] = zec.NewZCashRPC BlockChainFactories["Ethereum"] = eth.NewEthereumRPC + BlockChainFactories["Ethereum Classic"] = eth.NewEthereumRPC BlockChainFactories["Ethereum Testnet Ropsten"] = eth.NewEthereumRPC BlockChainFactories["Bcash"] = bch.NewBCashRPC BlockChainFactories["Bcash Testnet"] = bch.NewBCashRPC @@ -131,9 +132,9 @@ func (c *blockChainWithMetrics) GetSubversion() string { return c.b.GetSubversion() } -func (c *blockChainWithMetrics) GetBlockChainInfo() (v string, err error) { - defer func(s time.Time) { c.observeRPCLatency("GetBlockChainInfo", s, err) }(time.Now()) - return c.b.GetBlockChainInfo() +func (c *blockChainWithMetrics) GetChainInfo() (v *bchain.ChainInfo, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetChainInfo", s, err) }(time.Now()) + return c.b.GetChainInfo() } func (c *blockChainWithMetrics) GetBestBlockHash() (v string, err error) { @@ -161,6 +162,11 @@ func (c *blockChainWithMetrics) GetBlock(hash string, height uint32) (v *bchain. return c.b.GetBlock(hash, height) } +func (c *blockChainWithMetrics) GetBlockInfo(hash string) (v *bchain.BlockInfo, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetBlockInfo", s, err) }(time.Now()) + return c.b.GetBlockInfo(hash) +} + func (c *blockChainWithMetrics) GetMempool() (v []string, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempool", s, err) }(time.Now()) return c.b.GetMempool() @@ -191,7 +197,7 @@ func (c *blockChainWithMetrics) SendRawTransaction(tx string) (v string, err err return c.b.SendRawTransaction(tx) } -func (c *blockChainWithMetrics) ResyncMempool(onNewTxAddr func(txid string, addr string)) (count int, err error) { +func (c *blockChainWithMetrics) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (count int, err error) { defer func(s time.Time) { c.observeRPCLatency("ResyncMempool", s, err) }(time.Now()) count, err = c.b.ResyncMempool(onNewTxAddr) if err == nil { diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 02814f89..41f9f2ad 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -87,8 +87,22 @@ func (p *BitcoinParser) addressToOutputScript(address string) ([]byte, error) { // TryParseOPReturn tries to process OP_RETURN script and return its string representation func TryParseOPReturn(script []byte) string { if len(script) > 1 && script[0] == txscript.OP_RETURN { - l := int(script[1]) - data := script[2:] + // trying 2 variants of OP_RETURN data + // 1) OP_RETURN OP_PUSHDATA1 + // 2) OP_RETURN + var data []byte + var l int + if script[1] == txscript.OP_PUSHDATA1 && len(script) > 2 { + l = int(script[2]) + data = script[3:] + if l != len(data) { + l = int(script[1]) + data = script[2:] + } + } else { + l = int(script[1]) + data = script[2:] + } if l == len(data) { isASCII := true for _, c := range data { @@ -101,7 +115,7 @@ func TryParseOPReturn(script []byte) string { if isASCII { ed = "(" + string(data) + ")" } else { - ed = hex.EncodeToString([]byte{byte(l)}) + " " + hex.EncodeToString(data) + ed = hex.EncodeToString(data) } return "OP_RETURN " + ed } diff --git a/bchain/coins/btc/bitcoinparser_test.go b/bchain/coins/btc/bitcoinparser_test.go index 2350f3ef..80245180 100644 --- a/bchain/coins/btc/bitcoinparser_test.go +++ b/bchain/coins/btc/bitcoinparser_test.go @@ -108,10 +108,17 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { want2: false, wantErr: false, }, + { + name: "OP_RETURN OP_PUSHDATA1 ascii", + args: args{script: "6a4c0b446c6f7568792074657874"}, + want: []string{"OP_RETURN (Dlouhy text)"}, + want2: false, + wantErr: false, + }, { name: "OP_RETURN hex", args: args{script: "6a072020f1686f6a20"}, - want: []string{"OP_RETURN 07 2020f1686f6a20"}, + want: []string{"OP_RETURN 2020f1686f6a20"}, want2: false, wantErr: false, }, diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index ead2380c..27494586 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -102,10 +102,11 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT // and if successful it connects to ZeroMQ and creates mempool handler func (b *BitcoinRPC) GetChainInfoAndInitializeMempool(bc bchain.BlockChain) (string, error) { // try to connect to block chain and get some info - chainName, err := bc.GetBlockChainInfo() + ci, err := bc.GetChainInfo() if err != nil { return "", err } + chainName := ci.Chain mq, err := bchain.NewMQ(b.ChainConfig.MessageQueueBinding, b.pushHandler) if err != nil { @@ -217,10 +218,30 @@ type CmdGetBlockChainInfo struct { type ResGetBlockChainInfo struct { Error *bchain.RPCError `json:"error"` Result struct { - Chain string `json:"chain"` - Blocks int `json:"blocks"` - Headers int `json:"headers"` - Bestblockhash string `json:"bestblockhash"` + Chain string `json:"chain"` + Blocks int `json:"blocks"` + Headers int `json:"headers"` + Bestblockhash string `json:"bestblockhash"` + Difficulty json.Number `json:"difficulty"` + SizeOnDisk int64 `json:"size_on_disk"` + Warnings string `json:"warnings"` + } `json:"result"` +} + +// getnetworkinfo + +type CmdGetNetworkInfo struct { + Method string `json:"method"` +} + +type ResGetNetworkInfo struct { + Error *bchain.RPCError `json:"error"` + Result struct { + Version json.Number `json:"version"` + Subversion json.Number `json:"subversion"` + ProtocolVersion json.Number `json:"protocolversion"` + Timeoffset float64 `json:"timeoffset"` + Warnings string `json:"warnings"` } `json:"result"` } @@ -265,9 +286,14 @@ type ResGetBlockRaw struct { Result string `json:"result"` } +type BlockThin struct { + bchain.BlockHeader + Txids []string `json:"tx"` +} + type ResGetBlockThin struct { Error *bchain.RPCError `json:"error"` - Result bchain.ThinBlock `json:"result"` + Result BlockThin `json:"result"` } type ResGetBlockFull struct { @@ -275,6 +301,11 @@ type ResGetBlockFull struct { Result bchain.Block `json:"result"` } +type ResGetBlockInfo struct { + Error *bchain.RPCError `json:"error"` + Result bchain.BlockInfo `json:"result"` +} + // getrawtransaction type CmdGetRawTransaction struct { @@ -386,21 +417,48 @@ func (b *BitcoinRPC) GetBestBlockHeight() (uint32, error) { return res.Result, nil } -// GetBlockChainInfo returns the name of the block chain: main/test/regtest. -func (b *BitcoinRPC) GetBlockChainInfo() (string, error) { +// GetChainInfo returns information about the connected backend +func (b *BitcoinRPC) GetChainInfo() (*bchain.ChainInfo, error) { glog.V(1).Info("rpc: getblockchaininfo") - res := ResGetBlockChainInfo{} - req := CmdGetBlockChainInfo{Method: "getblockchaininfo"} - err := b.Call(&req, &res) - + resCi := ResGetBlockChainInfo{} + err := b.Call(&CmdGetBlockChainInfo{Method: "getblockchaininfo"}, &resCi) if err != nil { - return "", err + return nil, err } - if res.Error != nil { - return "", res.Error + if resCi.Error != nil { + return nil, resCi.Error } - return res.Result.Chain, nil + + glog.V(1).Info("rpc: getnetworkinfo") + resNi := ResGetNetworkInfo{} + err = b.Call(&CmdGetNetworkInfo{Method: "getnetworkinfo"}, &resNi) + if err != nil { + return nil, err + } + if resNi.Error != nil { + return nil, resNi.Error + } + + rv := &bchain.ChainInfo{ + Bestblockhash: resCi.Result.Bestblockhash, + Blocks: resCi.Result.Blocks, + Chain: resCi.Result.Chain, + Difficulty: string(resCi.Result.Difficulty), + Headers: resCi.Result.Headers, + SizeOnDisk: resCi.Result.SizeOnDisk, + Subversion: string(resNi.Result.Subversion), + Timeoffset: resNi.Result.Timeoffset, + } + rv.Version = string(resNi.Result.Version) + rv.ProtocolVersion = string(resNi.Result.ProtocolVersion) + if len(resCi.Result.Warnings) > 0 { + rv.Warnings = resCi.Result.Warnings + " " + } + if resCi.Result.Warnings != resNi.Result.Warnings { + rv.Warnings += resNi.Result.Warnings + } + return rv, nil } func isErrBlockNotFound(err *bchain.RPCError) bool { @@ -454,7 +512,7 @@ func (b *BitcoinRPC) GetBlockHeader(hash string) (*bchain.BlockHeader, error) { // GetBlock returns block with given hash. func (b *BitcoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { var err error - if hash == "" && height > 0 { + if hash == "" { hash, err = b.GetBlockHash(height) if err != nil { return nil, err @@ -483,7 +541,29 @@ func (b *BitcoinRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) return block, nil } -// getBlockWithoutHeader is an optimization - it does not call GetBlockHeader to get prev, next hashes +// GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids +func (b *BitcoinRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + glog.V(1).Info("rpc: getblock (verbosity=1) ", hash) + + res := ResGetBlockInfo{} + req := CmdGetBlock{Method: "getblock"} + req.Params.BlockHash = hash + req.Params.Verbosity = 1 + err := b.Call(&req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + if isErrBlockNotFound(res.Error) { + return nil, bchain.ErrBlockNotFound + } + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + return &res.Result, nil +} + +// GetBlockWithoutHeader is an optimization - it does not call GetBlockHeader to get prev, next hashes // instead it sets to header only block hash and height passed in parameters func (b *BitcoinRPC) GetBlockWithoutHeader(hash string, height uint32) (*bchain.Block, error) { data, err := b.GetBlockRaw(hash) @@ -613,7 +693,7 @@ func (b *BitcoinRPC) GetTransaction(txid string) (*bchain.Tx, error) { // ResyncMempool gets mempool transactions and maps output scripts to transactions. // ResyncMempool is not reentrant, it should be called from a single thread. // It returns number of transactions in mempool -func (b *BitcoinRPC) ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) { +func (b *BitcoinRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { return b.Mempool.Resync(onNewTxAddr) } diff --git a/bchain/coins/btg/bgoldparser.go b/bchain/coins/btg/bgoldparser.go index ab4e38b5..1e87aa93 100644 --- a/bchain/coins/btg/bgoldparser.go +++ b/bchain/coins/btg/bgoldparser.go @@ -5,6 +5,7 @@ import ( "blockbook/bchain/coins/btc" "blockbook/bchain/coins/utils" "bytes" + "encoding/binary" "io" "github.com/btcsuite/btcd/chaincfg/chainhash" @@ -22,7 +23,7 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic @@ -62,6 +63,9 @@ func NewBGoldParser(params *chaincfg.Params, c *btc.Configuration) *BGoldParser // the regression test Bitcoin Cash network, the test Bitcoin Cash network and // the simulation test Bitcoin Cash network, in this order func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams @@ -75,11 +79,13 @@ func GetChainParams(chain string) *chaincfg.Params { // headerFixedLength is the length of fixed fields of a block (i.e. without solution) // see https://github.com/BTCGPU/BTCGPU/wiki/Technical-Spec#block-header const headerFixedLength = 44 + (chainhash.HashSize * 3) +const timestampOffset = 100 +const timestampLength = 4 // ParseBlock parses raw block to our Block struct func (p *BGoldParser) ParseBlock(b []byte) (*bchain.Block, error) { r := bytes.NewReader(b) - err := skipHeader(r, 0) + time, err := getTimestampAndSkipHeader(r, 0) if err != nil { return nil, err } @@ -95,24 +101,41 @@ func (p *BGoldParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: time, + }, + Txs: txs, + }, nil } -func skipHeader(r io.ReadSeeker, pver uint32) error { - _, err := r.Seek(headerFixedLength, io.SeekStart) +func getTimestampAndSkipHeader(r io.ReadSeeker, pver uint32) (int64, error) { + _, err := r.Seek(timestampOffset, io.SeekStart) if err != nil { - return err + return 0, err + } + + buf := make([]byte, timestampLength) + if _, err = io.ReadFull(r, buf); err != nil { + return 0, err + } + time := binary.LittleEndian.Uint32(buf) + + _, err = r.Seek(headerFixedLength-timestampOffset-timestampLength, io.SeekCurrent) + if err != nil { + return 0, err } size, err := wire.ReadVarInt(r, pver) if err != nil { - return err + return 0, err } _, err = r.Seek(int64(size), io.SeekCurrent) if err != nil { - return err + return 0, err } - return nil + return int64(time), nil } diff --git a/bchain/coins/btg/bgoldparser_test.go b/bchain/coins/btg/bgoldparser_test.go index 8834cfc0..e37fecd7 100644 --- a/bchain/coins/btg/bgoldparser_test.go +++ b/bchain/coins/btg/bgoldparser_test.go @@ -12,72 +12,86 @@ import ( "testing" ) -var testParseBlockTxs = map[int][]string{ - 104000: []string{ - "331d4ef64118e9e5be75f0f51f1a4c5057550c3320e22ff7206f3e1101f113d0", - "1f4817d8e91c21d8c8d163dabccdd1875f760fd2dc34a1c2b7b8fa204e103597", - "268163b1a1092aa0996d118a6027b0b6f1076627e02addc4e66ae30239936818", - "27277a1049fafa2a46368ad02961d37da633416b014bcd42a1f1391753cbf559", - "276d2d331585d0968762d9788f71ae71524ccba3494f638b2328ac51e52edd3d", - "28d9f85089834c20507cc5e4ec54aaaf5b79feab80cad24a48b8296b6d327a43", - "2a2d66d3d9e8b6f154958a388377f681abd82ce98785a5bbf2e27d0ca454da3f", - "39c9e995a12b638b541d21ed9f9e555716709add6a97c8ba63fe26e4d26bdc3f", - "4a768efc6cc0716932800115989376c2ce3e5e17668b08bd2f43031105a6ac6e", - "4bc41fa0188d988d853a176eb847a541c5adf35348eab21d128904aab36e8574", - "4cad1bd38cc7be880f3a968af6f13a3aeb5dbdb51a774b7d1ae3d6d6bfd114e4", - "6bc558801583bfdb656106889c4b8bd783168133784666338c57e5b2a675a922", - "75eb5c1aa89b18ce71f363f95147942b46645ca9b1e472784fcb8c9a096f4f5c", - "91755365cff22c4eed09a57a8fb7b2faa5c1caa5fa750c89f830a2bb56fa4c68", - "9417d34969891f2a0b9aa3a1226505edf3b429fa1acd21a140d358a217d11d55", - "950fbb5f413af9f7c6e5dabfacf68b715c9851b5cf6ab6806960f7cc7cad2f9d", - "99b134dae55ddfab1f5d5243c2e60030a9ed969ba5915f98840b877f8af28ce0", - "9d7b15eaaccce66e25efe7e2979454ae4968b61281d50f32be9872d2d256c244", - "a54df5296f1d1a6101cee0869ffda03502e417fc72475b7902a6dfd5b9329399", - "adba400f14b476f0c2b11340ee1fa440208b49fd99c1072136198b5c43664826", - "bd7d8fee068bd45b06b4c17ccdf577b4bc2b21c9c4e0cee8453409b0e63b4f5d", - "beabd2d68b66a9b47b6aff23b569c1b59e429074f06bdd4993e9d3c2cb69c992", - "bfa81174d549eb7ed15be3f965686aff3084f22523c52fbed03c3fc3e18b7cda", - "e42472099cb039b1c2248b611ce212580338550977e02bd77accdf29bfd86e96", - "f056e02b12d99377f724a4987cde68ecf6f234fd7e2bdf4324172c03d015ba25", - "f1815cfb1ef4cfe13ff5ec2c15b5bc55fde043db85daca1bb34cc1b491193308", - "f225abce64f75383686fa08abe47242e59e97809a31c8fd7a88acce1e6cbcd27", - "f93f1b125bfa2da5ccaaf30ff96635b905b657d48a5962c24be93884a82ef354", - "fef75d015f2e9926d1d4bf82e567b91e51af66a6e636d03a072f203dd3062ae5", - "051b60a6accead85da54b8d18f4b2360ea946da948d3a27836306d2927fed13e", - "28e47b050ec4064cdbd3364f3be9445d52635e9730691ef71ed1db0f0147afd6", - "446ebde2457102bcbc2c86cac6ff582c595b00346fd0b27ea5a645240020504b", - "46c8fafe2b7bb1646aeefa229b18fa7ffe20fee0a30c4a9ef3e63c36c808f6f7", - "550d96cf82fbe91dcc9b96e7aa404f392ee47400c22a98a7631d29eee43fceaa", - "59b6b78a72cc33cd29741894b3007b1330fc7f7945bdc0a7a4044ed1dd289c19", - "5a3aa07474338cf193c1d7aacdc37f3311c971857ba8cfd308e8fabf5e473882", - "82e014b1a9c6cb7729274653ce748c66953de6abb3d1411471515b41b727cf75", - "8d70af4f135696da396c9aa9f936b54195bfbe6ff0e08b3b210ca0b52bc167d2", - "9949c2f2f3b96a557ef6e14004cbd239a0744c056faca34bdff01e125b4380e8", - "d09a8c83123ce1cb6ff837e7670aab5f50c5155d9706dd26f7e0761fd20c5536", - "f601482efc5b2dd3d0031e318e840cd06f7cab0ffff8cc37a5bf471b515ddfb7", - "f88d3c0ebe8b294f11e70d2ae6d2f0048437bfb20dae7e69d545a4c72d3432dd", - "2b9e574b90556250b20a79d9c94ceaff3dfd062291c34c3fa79c7ca8d85a3500", - "b9484ef8e38ceafe8d2b09ecf59562d262b15d185844b2d00db362718d52b2c2", - "07a4af0a81b55313a1c16da7d808829d689436fd078fa9559b6d1603dd72474e", - "3393bdcc3a7232b37d0fb6b56d603a2b9b0419e461bf989f1c375859a5d0156a", - "33ad36d79d63b575c7532c516f16b19541f5c637caf7073beb7ddf604c3f39cc", +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ + 104000: testBlock{ + size: 15776, + time: 1295705889, + txs: []string{ + "331d4ef64118e9e5be75f0f51f1a4c5057550c3320e22ff7206f3e1101f113d0", + "1f4817d8e91c21d8c8d163dabccdd1875f760fd2dc34a1c2b7b8fa204e103597", + "268163b1a1092aa0996d118a6027b0b6f1076627e02addc4e66ae30239936818", + "27277a1049fafa2a46368ad02961d37da633416b014bcd42a1f1391753cbf559", + "276d2d331585d0968762d9788f71ae71524ccba3494f638b2328ac51e52edd3d", + "28d9f85089834c20507cc5e4ec54aaaf5b79feab80cad24a48b8296b6d327a43", + "2a2d66d3d9e8b6f154958a388377f681abd82ce98785a5bbf2e27d0ca454da3f", + "39c9e995a12b638b541d21ed9f9e555716709add6a97c8ba63fe26e4d26bdc3f", + "4a768efc6cc0716932800115989376c2ce3e5e17668b08bd2f43031105a6ac6e", + "4bc41fa0188d988d853a176eb847a541c5adf35348eab21d128904aab36e8574", + "4cad1bd38cc7be880f3a968af6f13a3aeb5dbdb51a774b7d1ae3d6d6bfd114e4", + "6bc558801583bfdb656106889c4b8bd783168133784666338c57e5b2a675a922", + "75eb5c1aa89b18ce71f363f95147942b46645ca9b1e472784fcb8c9a096f4f5c", + "91755365cff22c4eed09a57a8fb7b2faa5c1caa5fa750c89f830a2bb56fa4c68", + "9417d34969891f2a0b9aa3a1226505edf3b429fa1acd21a140d358a217d11d55", + "950fbb5f413af9f7c6e5dabfacf68b715c9851b5cf6ab6806960f7cc7cad2f9d", + "99b134dae55ddfab1f5d5243c2e60030a9ed969ba5915f98840b877f8af28ce0", + "9d7b15eaaccce66e25efe7e2979454ae4968b61281d50f32be9872d2d256c244", + "a54df5296f1d1a6101cee0869ffda03502e417fc72475b7902a6dfd5b9329399", + "adba400f14b476f0c2b11340ee1fa440208b49fd99c1072136198b5c43664826", + "bd7d8fee068bd45b06b4c17ccdf577b4bc2b21c9c4e0cee8453409b0e63b4f5d", + "beabd2d68b66a9b47b6aff23b569c1b59e429074f06bdd4993e9d3c2cb69c992", + "bfa81174d549eb7ed15be3f965686aff3084f22523c52fbed03c3fc3e18b7cda", + "e42472099cb039b1c2248b611ce212580338550977e02bd77accdf29bfd86e96", + "f056e02b12d99377f724a4987cde68ecf6f234fd7e2bdf4324172c03d015ba25", + "f1815cfb1ef4cfe13ff5ec2c15b5bc55fde043db85daca1bb34cc1b491193308", + "f225abce64f75383686fa08abe47242e59e97809a31c8fd7a88acce1e6cbcd27", + "f93f1b125bfa2da5ccaaf30ff96635b905b657d48a5962c24be93884a82ef354", + "fef75d015f2e9926d1d4bf82e567b91e51af66a6e636d03a072f203dd3062ae5", + "051b60a6accead85da54b8d18f4b2360ea946da948d3a27836306d2927fed13e", + "28e47b050ec4064cdbd3364f3be9445d52635e9730691ef71ed1db0f0147afd6", + "446ebde2457102bcbc2c86cac6ff582c595b00346fd0b27ea5a645240020504b", + "46c8fafe2b7bb1646aeefa229b18fa7ffe20fee0a30c4a9ef3e63c36c808f6f7", + "550d96cf82fbe91dcc9b96e7aa404f392ee47400c22a98a7631d29eee43fceaa", + "59b6b78a72cc33cd29741894b3007b1330fc7f7945bdc0a7a4044ed1dd289c19", + "5a3aa07474338cf193c1d7aacdc37f3311c971857ba8cfd308e8fabf5e473882", + "82e014b1a9c6cb7729274653ce748c66953de6abb3d1411471515b41b727cf75", + "8d70af4f135696da396c9aa9f936b54195bfbe6ff0e08b3b210ca0b52bc167d2", + "9949c2f2f3b96a557ef6e14004cbd239a0744c056faca34bdff01e125b4380e8", + "d09a8c83123ce1cb6ff837e7670aab5f50c5155d9706dd26f7e0761fd20c5536", + "f601482efc5b2dd3d0031e318e840cd06f7cab0ffff8cc37a5bf471b515ddfb7", + "f88d3c0ebe8b294f11e70d2ae6d2f0048437bfb20dae7e69d545a4c72d3432dd", + "2b9e574b90556250b20a79d9c94ceaff3dfd062291c34c3fa79c7ca8d85a3500", + "b9484ef8e38ceafe8d2b09ecf59562d262b15d185844b2d00db362718d52b2c2", + "07a4af0a81b55313a1c16da7d808829d689436fd078fa9559b6d1603dd72474e", + "3393bdcc3a7232b37d0fb6b56d603a2b9b0419e461bf989f1c375859a5d0156a", + "33ad36d79d63b575c7532c516f16b19541f5c637caf7073beb7ddf604c3f39cc", + }, }, - 532144: []string{ - "574348e23301cc89535408b6927bf75f2ac88fadf8fdfb181c17941a5de02fe0", - "9f048446401e7fac84963964df045b1f3992eda330a87b02871e422ff0a3fd28", - "9516c320745a227edb07c98087b1febea01c3ba85122a34387896fc82ba965e4", - "9d37e1ab5a28c49ce5e7ece4a2b9df740fb4c3a84bdec93b3023148cf20f0de7", - "a3cd0481b983ba402fed8805ef9daf5063d6d4e5085b82eca5b4781c9e362d6a", - "7f2c2567e8de0321744817cfeb751922d7e8d82ef71aa01164c84fb74a463a53", - "cd064315e3f5d07920b3d159160c218d0bb5b7b4be606265767b208ae7e256eb", - "a9523400f341aa425b0fcc00656ec1fa5421bf3545433bff98a8614fc9a99d1f", - "ec766daacbb05a8f48a3205e5c6494a7c817bd35eefff9aaf55e0dd47fe6e8fc", - "0837a4116872abf52caa52d1ff7608674ba5b09a239a1f43f3a25ba4052e4c77", - "a3e23a0344fe6ba7083fc6afb940517cdb666dce00389cbd8598bd599199cdda", - "048d951cef84d35d68f0bc3b575662caf23fee692e8035bd5efe38ab67e0d6c2", - "11307491b24d42ddd7ea27fc795d444b65c3936be31b904a97da68fabc85e5b8", - "84ad99dc0884e03fc71f163eebf515a1eb79d00b1aad7a1126b22747960a8275", - "728c8d0858e506d4a1a9b506f7b974b335e6c54047af9f40d4cb1a0561f783e1", + 532144: testBlock{ + size: 12198, + time: 1528372417, + txs: []string{ + "574348e23301cc89535408b6927bf75f2ac88fadf8fdfb181c17941a5de02fe0", + "9f048446401e7fac84963964df045b1f3992eda330a87b02871e422ff0a3fd28", + "9516c320745a227edb07c98087b1febea01c3ba85122a34387896fc82ba965e4", + "9d37e1ab5a28c49ce5e7ece4a2b9df740fb4c3a84bdec93b3023148cf20f0de7", + "a3cd0481b983ba402fed8805ef9daf5063d6d4e5085b82eca5b4781c9e362d6a", + "7f2c2567e8de0321744817cfeb751922d7e8d82ef71aa01164c84fb74a463a53", + "cd064315e3f5d07920b3d159160c218d0bb5b7b4be606265767b208ae7e256eb", + "a9523400f341aa425b0fcc00656ec1fa5421bf3545433bff98a8614fc9a99d1f", + "ec766daacbb05a8f48a3205e5c6494a7c817bd35eefff9aaf55e0dd47fe6e8fc", + "0837a4116872abf52caa52d1ff7608674ba5b09a239a1f43f3a25ba4052e4c77", + "a3e23a0344fe6ba7083fc6afb940517cdb666dce00389cbd8598bd599199cdda", + "048d951cef84d35d68f0bc3b575662caf23fee692e8035bd5efe38ab67e0d6c2", + "11307491b24d42ddd7ea27fc795d444b65c3936be31b904a97da68fabc85e5b8", + "84ad99dc0884e03fc71f163eebf515a1eb79d00b1aad7a1126b22747960a8275", + "728c8d0858e506d4a1a9b506f7b974b335e6c54047af9f40d4cb1a0561f783e1", + }, }, } @@ -104,7 +118,7 @@ func helperLoadBlock(t *testing.T, height int) []byte { func TestParseBlock(t *testing.T) { p := NewBGoldParser(GetChainParams("main"), &btc.Configuration{}) - for height, txs := range testParseBlockTxs { + for height, tb := range testParseBlockTxs { b := helperLoadBlock(t, height) blk, err := p.ParseBlock(b) @@ -112,11 +126,19 @@ func TestParseBlock(t *testing.T) { t.Fatal(err) } - if len(blk.Txs) != len(txs) { - t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(txs)) + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) } - for ti, tx := range txs { + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { if blk.Txs[ti].Txid != tx { t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) } diff --git a/bchain/coins/dash/dashparser.go b/bchain/coins/dash/dashparser.go index 9b93e41b..d2371ef5 100644 --- a/bchain/coins/dash/dashparser.go +++ b/bchain/coins/dash/dashparser.go @@ -19,7 +19,7 @@ var ( RegtestParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic @@ -67,6 +67,9 @@ func NewDashParser(params *chaincfg.Params, c *btc.Configuration) *DashParser { // the regression test Dash network, the test Dash network and // the simulation test Dash network, in this order func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/dogecoin/dogecoinparser.go b/bchain/coins/dogecoin/dogecoinparser.go index 6975c688..56c95983 100644 --- a/bchain/coins/dogecoin/dogecoinparser.go +++ b/bchain/coins/dogecoin/dogecoinparser.go @@ -18,7 +18,7 @@ var ( MainNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic MainNetParams.PubKeyHashAddrID = []byte{30} @@ -43,6 +43,9 @@ func NewDogecoinParser(params *chaincfg.Params, c *btc.Configuration) *DogecoinP // GetChainParams contains network parameters for the main Dogecoin network, // and the test Dogecoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { default: return &MainNetParams @@ -75,5 +78,11 @@ func (p *DogecoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: h.Timestamp.Unix(), + }, + Txs: txs, + }, nil } diff --git a/bchain/coins/dogecoin/dogecoinparser_test.go b/bchain/coins/dogecoin/dogecoinparser_test.go index f9c1037e..5638f664 100644 --- a/bchain/coins/dogecoin/dogecoinparser_test.go +++ b/bchain/coins/dogecoin/dogecoinparser_test.go @@ -259,89 +259,111 @@ func Test_UnpackTx(t *testing.T) { } } -var testParseBlockTxs = map[int][]string{ +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ // block without auxpow - 12345: []string{ - "9d1662dcc1443af9999c4fd1d6921b91027b5e2d0d3ebfaa41d84163cb99cad5", - "8284292cedeb0c9c509f9baa235802d52a546e1e9990040d35d018b97ad11cfa", - "3299d93aae5c3d37c795c07150ceaf008aefa5aad3205ea2519f94a35adbbe10", - "3f03016f32b63db48fdc0b17443c2d917ba5e307dcc2fc803feeb21c7219ee1b", - "a889449e9bc618c131c01f564cd309d2217ba1c5731480314795e44f1e02609b", - "29f79d91c10bc311ff5b69fe7ba57101969f68b6391cf0ca67d5f37ca1f0601b", - "b794ebc7c0176c35b125cd8b84a980257cf3dd9cefe2ed47da4ed1d73ee568f3", - "0ec479ba3c954dd422d75c4c5488a6edc3c588deb10ebdbfa8bd8edb7afcfea0", - "f357b6e667dfa456e7988bfa474377df25d0e0bfe07e5f97fc97ea3a0155f031", - "4ff189766f0455721a93d6be27a91eafa750383c800cb053fad2f86c434122d2", - "446d164e2ec4c9f2ac6c499c110735606d949a3625fb849274ac627c033eddbc", - "c489edebd8a2e17fd08f2801f528b95663aaafe15c897d56686423dd430e2d1f", - "3f42a7f1a356897da324d41eed94169c79438212bb9874eea58e9cbaf07481df", - "62c88fdd0fb111676844fcbaebc9e2211a0c990aa7e7529539cb25947a307a1b", - "522c47e315bc1949826339c535d419eb206aec4a332f91dfbd25c206f3c9527b", - "18ea78346e7e34cbdf2d2b6ba1630f8b15f9ef9a940114a3e6ee92d26f96691e", - "43dc0fbd1b9b87bcfc9a51c89457a7b3274855c01d429193aff1181791225f3c", - "d78cdfaadbe5b6b591529cb5c6869866a4cabe46ef82aa835fd2432056b4a383", - "d181759c7a3900ccaf4958f1f25a44949163ceefc306006502efc7a1de6f579e", - "8610b9230188854c7871258163cd1c2db353443d631c5512bff17224a24e95bf", - "e82f40a6bea32122f1d568d427c92708dcb684bdb3035ff3905617230e5ae5b8", - "c50ae6c127f8c346c60e7438fbd10c44c3629f3fe426646db77a2250fb2939f9", - "585202c03894ecaf25188ba4e5447dadd413f2010c2dc2a65c37598dbc6ad907", - "8bd766fde8c65e2f724dad581944dde4e23e4dbb4f7f7faf55bc348923f4d5ee", - "2d2fa25691088181569e508dd8f683b21f2b80ceefb5ccbd6714ebe2a697139f", - "5954622ffc602bec177d61da6c26a68990c42c1886627b218c3ab0e9e3491f4a", - "01b634bc53334df1cd9f04522729a34d811c418c2535144c3ed156cbc319e43e", - "c429a6c8265482b2d824af03afe1c090b233a856f243791485cb4269f2729649", - "dbe79231b916b6fb47a91ef874f35150270eb571af60c2d640ded92b41749940", - "1c396493a8dfd59557052b6e8643123405894b64f48b2eb6eb7a003159034077", - "2e2816ffb7bf1378f11acf5ba30d498efc8fd219d4b67a725e8254ce61b1b7ee", + 12345: testBlock{ + size: 8582, + time: 1387104223, + txs: []string{ + "9d1662dcc1443af9999c4fd1d6921b91027b5e2d0d3ebfaa41d84163cb99cad5", + "8284292cedeb0c9c509f9baa235802d52a546e1e9990040d35d018b97ad11cfa", + "3299d93aae5c3d37c795c07150ceaf008aefa5aad3205ea2519f94a35adbbe10", + "3f03016f32b63db48fdc0b17443c2d917ba5e307dcc2fc803feeb21c7219ee1b", + "a889449e9bc618c131c01f564cd309d2217ba1c5731480314795e44f1e02609b", + "29f79d91c10bc311ff5b69fe7ba57101969f68b6391cf0ca67d5f37ca1f0601b", + "b794ebc7c0176c35b125cd8b84a980257cf3dd9cefe2ed47da4ed1d73ee568f3", + "0ec479ba3c954dd422d75c4c5488a6edc3c588deb10ebdbfa8bd8edb7afcfea0", + "f357b6e667dfa456e7988bfa474377df25d0e0bfe07e5f97fc97ea3a0155f031", + "4ff189766f0455721a93d6be27a91eafa750383c800cb053fad2f86c434122d2", + "446d164e2ec4c9f2ac6c499c110735606d949a3625fb849274ac627c033eddbc", + "c489edebd8a2e17fd08f2801f528b95663aaafe15c897d56686423dd430e2d1f", + "3f42a7f1a356897da324d41eed94169c79438212bb9874eea58e9cbaf07481df", + "62c88fdd0fb111676844fcbaebc9e2211a0c990aa7e7529539cb25947a307a1b", + "522c47e315bc1949826339c535d419eb206aec4a332f91dfbd25c206f3c9527b", + "18ea78346e7e34cbdf2d2b6ba1630f8b15f9ef9a940114a3e6ee92d26f96691e", + "43dc0fbd1b9b87bcfc9a51c89457a7b3274855c01d429193aff1181791225f3c", + "d78cdfaadbe5b6b591529cb5c6869866a4cabe46ef82aa835fd2432056b4a383", + "d181759c7a3900ccaf4958f1f25a44949163ceefc306006502efc7a1de6f579e", + "8610b9230188854c7871258163cd1c2db353443d631c5512bff17224a24e95bf", + "e82f40a6bea32122f1d568d427c92708dcb684bdb3035ff3905617230e5ae5b8", + "c50ae6c127f8c346c60e7438fbd10c44c3629f3fe426646db77a2250fb2939f9", + "585202c03894ecaf25188ba4e5447dadd413f2010c2dc2a65c37598dbc6ad907", + "8bd766fde8c65e2f724dad581944dde4e23e4dbb4f7f7faf55bc348923f4d5ee", + "2d2fa25691088181569e508dd8f683b21f2b80ceefb5ccbd6714ebe2a697139f", + "5954622ffc602bec177d61da6c26a68990c42c1886627b218c3ab0e9e3491f4a", + "01b634bc53334df1cd9f04522729a34d811c418c2535144c3ed156cbc319e43e", + "c429a6c8265482b2d824af03afe1c090b233a856f243791485cb4269f2729649", + "dbe79231b916b6fb47a91ef874f35150270eb571af60c2d640ded92b41749940", + "1c396493a8dfd59557052b6e8643123405894b64f48b2eb6eb7a003159034077", + "2e2816ffb7bf1378f11acf5ba30d498efc8fd219d4b67a725e8254ce61b1b7ee", + }, }, // 1st block with auxpow - 371337: []string{ - "4547b14bc16db4184fa9f141d645627430dd3dfa662d0e6f418fba497091da75", - "a965dba2ed06827ed9a24f0568ec05b73c431bc7f0fb6913b144e62db7faa519", - "5e3ab18cb7ba3abc44e62fb3a43d4c8168d00cf0a2e0f8dbeb2636bb9a212d12", - "f022935ac7c4c734bd2c9c6a780f8e7280352de8bd358d760d0645b7fe734a93", - "ec063cc8025f9f30a6ed40fc8b1fe63b0cbd2ea2c62664eb26b365e6243828ca", - "02c16e3389320da3e77686d39773dda65a1ecdf98a2ef9cfb938c9f4b58f7a40", + 371337: testBlock{ + size: 1704, + time: 1410464577, + txs: []string{ + "4547b14bc16db4184fa9f141d645627430dd3dfa662d0e6f418fba497091da75", + "a965dba2ed06827ed9a24f0568ec05b73c431bc7f0fb6913b144e62db7faa519", + "5e3ab18cb7ba3abc44e62fb3a43d4c8168d00cf0a2e0f8dbeb2636bb9a212d12", + "f022935ac7c4c734bd2c9c6a780f8e7280352de8bd358d760d0645b7fe734a93", + "ec063cc8025f9f30a6ed40fc8b1fe63b0cbd2ea2c62664eb26b365e6243828ca", + "02c16e3389320da3e77686d39773dda65a1ecdf98a2ef9cfb938c9f4b58f7a40", + }, }, // block with auxpow - 567890: []string{ - "db20feea53be1f60848a66604d5bca63df62de4f6c66220f9c84436d788625a8", - "cf7e9e27c0f56f0b100eaf5c776ce106025e3412bd5927c6e1ce575500e24eaa", - "af84e010c1cf0bd927740d08e5e8163db45397b70f00df07aea5339c14d5f3aa", - "7362e25e8131255d101e5d874e6b6bb2faa7a821356cb041f1843d0901dffdbd", - "3b875344302e8893f6d5c9e7269d806ed27217ec67944940ae9048fc619bdae9", - "e3b95e269b7c251d87e8e241ea2a08a66ec14d12a1012762be368b3db55471e3", - "6ba3f95a37bcab5d0cb5b8bd2fe48040db0a6ae390f320d6dcc8162cc096ff8f", - "3211ccc66d05b10959fa6e56d1955c12368ea52b40303558b254d7dc22570382", - "54c1b279e78b924dfa15857c80131c3ddf835ab02f513dc03aa514f87b680493", + 567890: testBlock{ + size: 3833, + time: 1422855443, + txs: []string{ + "db20feea53be1f60848a66604d5bca63df62de4f6c66220f9c84436d788625a8", + "cf7e9e27c0f56f0b100eaf5c776ce106025e3412bd5927c6e1ce575500e24eaa", + "af84e010c1cf0bd927740d08e5e8163db45397b70f00df07aea5339c14d5f3aa", + "7362e25e8131255d101e5d874e6b6bb2faa7a821356cb041f1843d0901dffdbd", + "3b875344302e8893f6d5c9e7269d806ed27217ec67944940ae9048fc619bdae9", + "e3b95e269b7c251d87e8e241ea2a08a66ec14d12a1012762be368b3db55471e3", + "6ba3f95a37bcab5d0cb5b8bd2fe48040db0a6ae390f320d6dcc8162cc096ff8f", + "3211ccc66d05b10959fa6e56d1955c12368ea52b40303558b254d7dc22570382", + "54c1b279e78b924dfa15857c80131c3ddf835ab02f513dc03aa514f87b680493", + }, }, // recent block - 2264125: []string{ - "76f0126562c99e020b5fba41b68dd8141a4f21eef62012b76a1e0635092045e9", - "7bb6688bec16de94014574e3e1d3f6f5fb956530d6b179b28db367f1fd8ae099", - "d7e2ee30c3d179ac896651fc09c1396333f41d952d008af8d5d6665cbea377bf", - "8e4783878df782003c43d014fcbb9c57d2034dfd1d9fcd7319bb1a9f501dbbb7", - "8d2a4ae226b6f23eea545957be5d71c68cd08674d96a3502d4ca21ffadacb5a9", - "a0da2b49de881133655c54b1b5c23af443a71c2b937e2d9bbdf3f498247e6b7b", - "c780a19b9cf46ed70b53c5d5722e8d33951211a4051cb165b25fb0c22a4ae1ff", - "ce29c2644d642bb4fedd09d0840ed98c9945bf292967fede8fcc6b26054b4058", - "a360b0566f68c329e2757918f67ee6421d3d76f70f1b452cdd32266805986119", - "17e85bd33cc5fb5035e489c5188979f45e75e92d14221eca937e14f5f7d7b074", - "3973eb930fd2d0726abbd81912eae645384268cd3500b9ec84d806fdd65a426a", - "b91cc1c98e5c77e80eec9bf93e86af27f810b00dfbce3ee2646758797a28d5f2", - "1a8c7bd3389dcbbc1133ee600898ed9e082f7a9c75f9eb52f33940ed7c2247ef", - "9b1782449bbd3fc3014c363167777f7bdf41f5ef6db192fbda784b29603911b0", - "afab4bcdc1a32891d638579c3029ae49ee72be3303425c6d62e1f8eaebe0ce18", - "5f839f9cd5293c02ff4f7cf5589c53dec52adb42a077599dc7a2c5842a156ca9", - "756d2dfd1d2872ba2531fae3b8984008506871bec41d19cb299f5e0f216cfb9b", - "6aa82514ab7a9cc624fabf3d06ccbd46ecb4009b3c784768e6243d7840d4bf93", - "d1430b3f7ecf147534796c39ba631ea22ac03530e25b9428367c0dc381b10863", - "2aeb69b1eb9eef8039da6b97d7851e46f57325851e6998ef5a84fc9a826c2c74", - "fc61d13eef806af8da693cfa621fe92110694f1514567b186a35c54e7ef4a188", - "a02dd44e60ba62fa00c83a67116f8079bf71062939b207bee0808cb98b30cf22", - "279f97cfc606fe62777b44614ff28675ce661687904e068e3ec79f619c4fdae7", - "d515d271849717b091a9c46bf11c47efb9d975e72b668c137786a208cf0a9739", - "a800da44e6eed944043561fe22ee0a6e11341e6bc1a8ec2789b83930cc9b170e", + 2264125: testBlock{ + size: 8531, + time: 1529099968, + txs: []string{ + "76f0126562c99e020b5fba41b68dd8141a4f21eef62012b76a1e0635092045e9", + "7bb6688bec16de94014574e3e1d3f6f5fb956530d6b179b28db367f1fd8ae099", + "d7e2ee30c3d179ac896651fc09c1396333f41d952d008af8d5d6665cbea377bf", + "8e4783878df782003c43d014fcbb9c57d2034dfd1d9fcd7319bb1a9f501dbbb7", + "8d2a4ae226b6f23eea545957be5d71c68cd08674d96a3502d4ca21ffadacb5a9", + "a0da2b49de881133655c54b1b5c23af443a71c2b937e2d9bbdf3f498247e6b7b", + "c780a19b9cf46ed70b53c5d5722e8d33951211a4051cb165b25fb0c22a4ae1ff", + "ce29c2644d642bb4fedd09d0840ed98c9945bf292967fede8fcc6b26054b4058", + "a360b0566f68c329e2757918f67ee6421d3d76f70f1b452cdd32266805986119", + "17e85bd33cc5fb5035e489c5188979f45e75e92d14221eca937e14f5f7d7b074", + "3973eb930fd2d0726abbd81912eae645384268cd3500b9ec84d806fdd65a426a", + "b91cc1c98e5c77e80eec9bf93e86af27f810b00dfbce3ee2646758797a28d5f2", + "1a8c7bd3389dcbbc1133ee600898ed9e082f7a9c75f9eb52f33940ed7c2247ef", + "9b1782449bbd3fc3014c363167777f7bdf41f5ef6db192fbda784b29603911b0", + "afab4bcdc1a32891d638579c3029ae49ee72be3303425c6d62e1f8eaebe0ce18", + "5f839f9cd5293c02ff4f7cf5589c53dec52adb42a077599dc7a2c5842a156ca9", + "756d2dfd1d2872ba2531fae3b8984008506871bec41d19cb299f5e0f216cfb9b", + "6aa82514ab7a9cc624fabf3d06ccbd46ecb4009b3c784768e6243d7840d4bf93", + "d1430b3f7ecf147534796c39ba631ea22ac03530e25b9428367c0dc381b10863", + "2aeb69b1eb9eef8039da6b97d7851e46f57325851e6998ef5a84fc9a826c2c74", + "fc61d13eef806af8da693cfa621fe92110694f1514567b186a35c54e7ef4a188", + "a02dd44e60ba62fa00c83a67116f8079bf71062939b207bee0808cb98b30cf22", + "279f97cfc606fe62777b44614ff28675ce661687904e068e3ec79f619c4fdae7", + "d515d271849717b091a9c46bf11c47efb9d975e72b668c137786a208cf0a9739", + "a800da44e6eed944043561fe22ee0a6e11341e6bc1a8ec2789b83930cc9b170e", + }, }, } @@ -368,7 +390,7 @@ func helperLoadBlock(t *testing.T, height int) []byte { func TestParseBlock(t *testing.T) { p := NewDogecoinParser(GetChainParams("main"), &btc.Configuration{}) - for height, txs := range testParseBlockTxs { + for height, tb := range testParseBlockTxs { b := helperLoadBlock(t, height) blk, err := p.ParseBlock(b) @@ -376,11 +398,19 @@ func TestParseBlock(t *testing.T) { t.Fatal(err) } - if len(blk.Txs) != len(txs) { - t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(txs)) + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) } - for ti, tx := range txs { + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { if blk.Txs[ti].Txid != tx { t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) } diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index cb08b664..83be98bf 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "math/big" + "strconv" "sync" "time" @@ -41,9 +42,7 @@ type EthereumRPC struct { client *ethclient.Client rpc *rpc.Client timeout time.Duration - rpcURL string Parser *EthereumParser - CoinName string Testnet bool Network string Mempool *bchain.NonUTXOMempool @@ -54,6 +53,7 @@ type EthereumRPC struct { chanNewTx chan ethcommon.Hash newTxSubscription *rpc.ClientSubscription ChainConfig *Configuration + isETC bool } // NewEthereumRPC returns new EthRPC instance. @@ -80,6 +80,9 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification s.Parser = NewEthereumParser() s.timeout = time.Duration(c.RPCTimeout) * time.Second + // detect ethereum classic + s.isETC = s.ChainConfig.CoinName == "Ethereum Classic" + // new blocks notifications handling // the subscription is done in Initialize s.chanNewBlock = make(chan *ethtypes.Header) @@ -143,21 +146,25 @@ func (b *EthereumRPC) Initialize() error { } glog.Info("rpc: block chain ", b.Network) - // subscriptions - if err = b.subscribe(func() (*rpc.ClientSubscription, error) { - // invalidate the previous subscription - it is either the first one or there was an error - b.newBlockSubscription = nil - ctx, cancel := context.WithTimeout(context.Background(), b.timeout) - defer cancel() - sub, err := b.rpc.EthSubscribe(ctx, b.chanNewBlock, "newHeads") - if err != nil { - return nil, errors.Annotatef(err, "EthSubscribe newHeads") + if b.isETC { + glog.Info(b.ChainConfig.CoinName, " does not support subscription to newHeads") + } else { + // subscriptions + if err = b.subscribe(func() (*rpc.ClientSubscription, error) { + // invalidate the previous subscription - it is either the first one or there was an error + b.newBlockSubscription = nil + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + sub, err := b.rpc.EthSubscribe(ctx, b.chanNewBlock, "newHeads") + if err != nil { + return nil, errors.Annotatef(err, "EthSubscribe newHeads") + } + b.newBlockSubscription = sub + glog.Info("Subscribed to newHeads") + return sub, nil + }); err != nil { + return err } - b.newBlockSubscription = sub - glog.Info("Subscribed to newHeads") - return sub, nil - }); err != nil { - return err } if err = b.subscribe(func() (*rpc.ClientSubscription, error) { // invalidate the previous subscription - it is either the first one or there was an error @@ -252,16 +259,23 @@ func (b *EthereumRPC) GetSubversion() string { return "" } -// GetBlockChainInfo returns the NetworkID of the ethereum network -func (b *EthereumRPC) GetBlockChainInfo() (string, error) { +// GetChainInfo returns information about the connected backend +func (b *EthereumRPC) GetChainInfo() (*bchain.ChainInfo, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() - id, err := b.client.NetworkID(ctx) if err != nil { - return "", err + return nil, err } - return id.String(), nil + rv := &bchain.ChainInfo{} + idi := int(id.Uint64()) + if idi == 1 { + rv.Chain = "mainnet" + } else { + rv.Chain = "testnet " + strconv.Itoa(idi) + } + // TODO - return more information about the chain + return rv, nil } func (b *EthereumRPC) getBestHeader() (*ethtypes.Header, error) { @@ -410,6 +424,12 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error return &bbk, nil } +// GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids +func (b *EthereumRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + // TODO - implement + return nil, errors.New("Not implemented yet") +} + // GetTransactionForMempool returns a transaction by the transaction ID. // It could be optimized for mempool, i.e. without block time and confirmations func (b *EthereumRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { @@ -427,7 +447,11 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { } else if tx == nil { return nil, ethereum.NotFound } else if tx.R == "" { - return nil, errors.Annotatef(fmt.Errorf("server returned transaction without signature"), "txid %v", txid) + if !b.isETC { + return nil, errors.Annotatef(fmt.Errorf("server returned transaction without signature"), "txid %v", txid) + } else { + glog.Warning("server returned transaction without signature, txid ", txid) + } } var btx *bchain.Tx if tx.BlockNumber == "" { @@ -503,12 +527,28 @@ func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, return r, nil } -// SendRawTransaction sends raw transaction -func (b *EthereumRPC) SendRawTransaction(tx string) (string, error) { - return "", errors.New("SendRawTransaction: not implemented") +// SendRawTransaction sends raw transaction. +func (b *EthereumRPC) SendRawTransaction(hex string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + var raw json.RawMessage + err := b.rpc.CallContext(ctx, &raw, "eth_sendRawTransaction", hex) + if err != nil { + return "", err + } else if len(raw) == 0 { + return "", errors.New("SendRawTransaction: failed") + } + var result string + if err := json.Unmarshal(raw, &result); err != nil { + return "", errors.Annotatef(err, "raw result %v", raw) + } + if result == "" { + return "", errors.New("SendRawTransaction: failed, empty result") + } + return result, nil } -func (b *EthereumRPC) ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) { +func (b *EthereumRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { return b.Mempool.Resync(onNewTxAddr) } diff --git a/bchain/coins/litecoin/litecoinparser.go b/bchain/coins/litecoin/litecoinparser.go index d81faa4f..7a60d2ac 100644 --- a/bchain/coins/litecoin/litecoinparser.go +++ b/bchain/coins/litecoin/litecoinparser.go @@ -18,7 +18,7 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic MainNetParams.PubKeyHashAddrID = []byte{48} @@ -53,6 +53,9 @@ func NewLitecoinParser(params *chaincfg.Params, c *btc.Configuration) *LitecoinP // GetChainParams contains network parameters for the main Litecoin network, // and the test Litecoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/monacoin/monacoinparser.go b/bchain/coins/monacoin/monacoinparser.go index ac082016..d5a9de37 100644 --- a/bchain/coins/monacoin/monacoinparser.go +++ b/bchain/coins/monacoin/monacoinparser.go @@ -26,7 +26,7 @@ var ( MonaTestParams monacoinCfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic MainNetParams.PubKeyHashAddrID = []byte{50} @@ -71,6 +71,9 @@ func NewMonacoinParser(params *chaincfg.Params, c *btc.Configuration) *MonacoinP // GetChainParams contains network parameters for the main Monacoin network, // and the test Monacoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/monacoin/monacoinparser_test.go b/bchain/coins/monacoin/monacoinparser_test.go index 3b9e759e..7c95d9a2 100644 --- a/bchain/coins/monacoin/monacoinparser_test.go +++ b/bchain/coins/monacoin/monacoinparser_test.go @@ -170,7 +170,7 @@ func Test_GetAddressesFromAddrDesc(t *testing.T) { { name: "OP_RETURN hex", args: args{script: "6a072020f1686f6a20"}, - want: []string{"OP_RETURN 07 2020f1686f6a20"}, + want: []string{"OP_RETURN 2020f1686f6a20"}, want2: false, wantErr: false, }, diff --git a/bchain/coins/namecoin/namecoinparser.go b/bchain/coins/namecoin/namecoinparser.go index 270f1ed6..bca57e55 100644 --- a/bchain/coins/namecoin/namecoinparser.go +++ b/bchain/coins/namecoin/namecoinparser.go @@ -18,7 +18,7 @@ var ( MainNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic MainNetParams.PubKeyHashAddrID = []byte{52} @@ -43,6 +43,9 @@ func NewNamecoinParser(params *chaincfg.Params, c *btc.Configuration) *NamecoinP // GetChainParams contains network parameters for the main Namecoin network, // and the test Namecoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { default: return &MainNetParams @@ -75,5 +78,11 @@ func (p *NamecoinParser) ParseBlock(b []byte) (*bchain.Block, error) { txs[ti] = p.TxFromMsgTx(t, false) } - return &bchain.Block{Txs: txs}, nil + return &bchain.Block{ + BlockHeader: bchain.BlockHeader{ + Size: len(b), + Time: h.Timestamp.Unix(), + }, + Txs: txs, + }, nil } diff --git a/bchain/coins/namecoin/namecoinparser_test.go b/bchain/coins/namecoin/namecoinparser_test.go index dcf34b5f..8eff0e00 100644 --- a/bchain/coins/namecoin/namecoinparser_test.go +++ b/bchain/coins/namecoin/namecoinparser_test.go @@ -53,11 +53,21 @@ func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { } } -var testParseBlockTxs = map[int][]string{ - 40000: []string{ - "e193a821393b20b99f4a4e05a481368ef8a8cfd43d0c45bdad7f53bc9535e844", - "ddcbf95797e81dd04127885bd001e96695b717e11c52721f6e8ee53f6dea8a6f", - "31ff728f24200f59fa4958e6c26de03d172b320e6eef2b8abecf6f94d01dd4ae", +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ + 40000: testBlock{ + size: 1385, + time: 1327728573, + txs: []string{ + "e193a821393b20b99f4a4e05a481368ef8a8cfd43d0c45bdad7f53bc9535e844", + "ddcbf95797e81dd04127885bd001e96695b717e11c52721f6e8ee53f6dea8a6f", + "31ff728f24200f59fa4958e6c26de03d172b320e6eef2b8abecf6f94d01dd4ae", + }, }, } @@ -84,7 +94,7 @@ func helperLoadBlock(t *testing.T, height int) []byte { func TestParseBlock(t *testing.T) { p := NewNamecoinParser(GetChainParams("main"), &btc.Configuration{}) - for height, txs := range testParseBlockTxs { + for height, tb := range testParseBlockTxs { b := helperLoadBlock(t, height) blk, err := p.ParseBlock(b) @@ -92,11 +102,19 @@ func TestParseBlock(t *testing.T) { t.Fatal(err) } - if len(blk.Txs) != len(txs) { - t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(txs)) + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) } - for ti, tx := range txs { + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { if blk.Txs[ti].Txid != tx { t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) } diff --git a/bchain/coins/utils/parserutils.go b/bchain/coins/utils/parserutils.go index 4fbe422c..c59979ed 100644 --- a/bchain/coins/utils/parserutils.go +++ b/bchain/coins/utils/parserutils.go @@ -33,7 +33,7 @@ func DecodeTransactions(r io.Reader, pver uint32, enc wire.MessageEncoding, blk if txCount > maxTxPerBlock { str := fmt.Sprintf("too many transactions to fit into a block "+ "[count %d, max %d]", txCount, maxTxPerBlock) - return &wire.MessageError{Func: "btg.decodeTransactions", Description: str} + return &wire.MessageError{Func: "utils.decodeTransactions", Description: str} } blk.Transactions = make([]*wire.MsgTx, 0, txCount) diff --git a/bchain/coins/vertcoin/vertcoinparser.go b/bchain/coins/vertcoin/vertcoinparser.go index 59fdbdcf..ef7d82fc 100644 --- a/bchain/coins/vertcoin/vertcoinparser.go +++ b/bchain/coins/vertcoin/vertcoinparser.go @@ -18,7 +18,7 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic MainNetParams.PubKeyHashAddrID = []byte{71} @@ -53,6 +53,9 @@ func NewVertcoinParser(params *chaincfg.Params, c *btc.Configuration) *VertcoinP // GetChainParams contains network parameters for the main Vertcoin network, // and the test Vertcoin network func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } switch chain { case "test": return &TestNetParams diff --git a/bchain/coins/zec/zcashparser.go b/bchain/coins/zec/zcashparser.go index c6452fc7..ea4917c3 100644 --- a/bchain/coins/zec/zcashparser.go +++ b/bchain/coins/zec/zcashparser.go @@ -19,7 +19,7 @@ var ( TestNetParams chaincfg.Params ) -func init() { +func initParams() { MainNetParams = chaincfg.MainNetParams MainNetParams.Net = MainnetMagic @@ -63,6 +63,9 @@ func NewZCashParser(params *chaincfg.Params, c *btc.Configuration) *ZCashParser // the regression test ZCash network, the test ZCash network and // the simulation test ZCash network, in this order func GetChainParams(chain string) *chaincfg.Params { + if MainNetParams.Name == "" { + initParams() + } var params *chaincfg.Params switch chain { case "test": diff --git a/bchain/mempool_nonutxo.go b/bchain/mempool_nonutxo.go index 1930aaa6..28485dab 100644 --- a/bchain/mempool_nonutxo.go +++ b/bchain/mempool_nonutxo.go @@ -52,7 +52,7 @@ func (m *NonUTXOMempool) updateMappings(newTxToInputOutput map[string][]addrInde // Resync gets mempool transactions and maps outputs to transactions. // Resync is not reentrant, it should be called from a single thread. // Read operations (GetTransactions) are safe. -func (m *NonUTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int, error) { +func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { start := time.Now() glog.V(1).Info("Mempool: resync") txs, err := m.chain.GetMempool() @@ -84,7 +84,7 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { - onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0]) + onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0], true) } } for _, input := range tx.Vin { @@ -96,6 +96,9 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int continue } io = append(io, addrIndex{string(addrDesc), int32(^i)}) + if onNewTxAddr != nil { + onNewTxAddr(tx.Txid, a, false) + } } } } diff --git a/bchain/mempool_utxo.go b/bchain/mempool_utxo.go index f111b513..54017e00 100644 --- a/bchain/mempool_utxo.go +++ b/bchain/mempool_utxo.go @@ -31,7 +31,7 @@ type UTXOMempool struct { addrDescToTx map[string][]outpoint chanTxid chan string chanAddrIndex chan txidio - onNewTxAddr func(txid string, addr string) + onNewTxAddr OnNewTxAddrFunc } // NewUTXOMempool creates new mempool handler. @@ -133,7 +133,7 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if m.onNewTxAddr != nil && len(output.ScriptPubKey.Addresses) == 1 { - m.onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0]) + m.onNewTxAddr(tx.Txid, output.ScriptPubKey.Addresses[0], true) } } dispatched := 0 @@ -170,7 +170,7 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul // Resync gets mempool transactions and maps outputs to transactions. // Resync is not reentrant, it should be called from a single thread. // Read operations (GetTransactions) are safe. -func (m *UTXOMempool) Resync(onNewTxAddr func(txid string, addr string)) (int, error) { +func (m *UTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { start := time.Now() glog.V(1).Info("mempool: resync") m.onNewTxAddr = onNewTxAddr diff --git a/bchain/types.go b/bchain/types.go index 00630f13..4728a4bf 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -71,11 +71,7 @@ type Block struct { Txs []Tx `json:"tx"` } -type ThinBlock struct { - BlockHeader - Txids []string `json:"tx"` -} - +// BlockHeader contains limited data (as needed for indexing) from backend block header type BlockHeader struct { Hash string `json:"hash"` Prev string `json:"previousblockhash"` @@ -86,6 +82,17 @@ type BlockHeader struct { Time int64 `json:"time,omitempty"` } +// BlockInfo contains extended block header data and a list of block txids +type BlockInfo struct { + BlockHeader + Version json.Number `json:"version"` + MerkleRoot string `json:"merkleroot"` + Nonce json.Number `json:"nonce"` + Bits string `json:"bits"` + Difficulty json.Number `json:"difficulty"` + Txids []string `json:"tx,omitempty"` +} + type MempoolEntry struct { Size uint32 `json:"size"` FeeSat big.Int @@ -103,6 +110,20 @@ type MempoolEntry struct { Depends []string `json:"depends"` } +type ChainInfo struct { + Chain string `json:"chain"` + Blocks int `json:"blocks"` + Headers int `json:"headers"` + Bestblockhash string `json:"bestblockhash"` + Difficulty string `json:"difficulty"` + SizeOnDisk int64 `json:"size_on_disk"` + Version string `json:"version"` + Subversion string `json:"subversion"` + ProtocolVersion string `json:"protocolversion"` + Timeoffset float64 `json:"timeoffset"` + Warnings string `json:"warnings"` +} + type RPCError struct { Code int `json:"code"` Message string `json:"message"` @@ -119,6 +140,12 @@ func (ad AddressDescriptor) String() string { return "ad:" + hex.EncodeToString(ad) } +// OnNewBlockFunc is used to send notification about a new block +type OnNewBlockFunc func(hash string, height uint32) + +// OnNewTxAddrFunc is used to send notification about a new transaction/address +type OnNewTxAddrFunc func(txid string, addr string, isOutput bool) + // BlockChain defines common interface to block chain daemon type BlockChain interface { // life-cycle methods @@ -129,13 +156,14 @@ type BlockChain interface { GetNetworkName() string GetSubversion() string GetCoinName() string + GetChainInfo() (*ChainInfo, error) // requests - GetBlockChainInfo() (string, error) GetBestBlockHash() (string, error) GetBestBlockHeight() (uint32, error) GetBlockHash(height uint32) (string, error) GetBlockHeader(hash string) (*BlockHeader, error) GetBlock(hash string, height uint32) (*Block, error) + GetBlockInfo(hash string) (*BlockInfo, error) GetMempool() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) @@ -143,7 +171,7 @@ type BlockChain interface { EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) // mempool - ResyncMempool(onNewTxAddr func(txid string, addr string)) (int, error) + ResyncMempool(onNewTxAddr OnNewTxAddrFunc) (int, error) GetMempoolTransactions(address string) ([]string, error) GetMempoolTransactionsForAddrDesc(addrDesc AddressDescriptor) ([]string, error) GetMempoolEntry(txid string) (*MempoolEntry, error) diff --git a/blockbook.go b/blockbook.go index c4888ce1..99809cd9 100644 --- a/blockbook.go +++ b/blockbook.go @@ -1,6 +1,7 @@ package main import ( + "blockbook/api" "blockbook/bchain" "blockbook/bchain/coins" "blockbook/common" @@ -19,7 +20,7 @@ import ( "syscall" "time" - "github.com/erikdubbelboer/gspt" + // "github.com/erikdubbelboer/gspt" "github.com/golang/glog" "github.com/juju/errors" ) @@ -36,8 +37,9 @@ const storeInternalStatePeriodMs = 59699 var ( blockchain = flag.String("blockchaincfg", "", "path to blockchain RPC service configuration json file") - dbPath = flag.String("datadir", "./data", "path to database directory") - dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") + dbPath = flag.String("datadir", "./data", "path to database directory") + dbCache = flag.Int("dbcache", 1<<29, "size of the rocksdb cache") + dbMaxOpenFiles = flag.Int("dbmaxopenfiles", 1<<14, "max open files by rocksdb") blockFrom = flag.Int("blockheight", -1, "height of the starting block") blockUntil = flag.Int("blockuntil", -1, "height of the final block") @@ -49,17 +51,17 @@ var ( repair = flag.Bool("repair", false, "repair the database") prof = flag.String("prof", "", "http server binding [address]:port of the interface to profiling data /debug/pprof/ (default no profiling)") - syncChunk = flag.Int("chunk", 100, "block chunk size for processing") - syncWorkers = flag.Int("workers", 8, "number of workers to process blocks") + syncChunk = flag.Int("chunk", 100, "block chunk size for processing in bulk mode") + syncWorkers = flag.Int("workers", 8, "number of workers to process blocks in bulk mode") dryRun = flag.Bool("dryrun", false, "do not index blocks, only download") debugMode = flag.Bool("debug", false, "debug mode, return more verbose errors, reload templates on each request") internalBinding = flag.String("internal", "", "internal http server binding [address]:port, (default no internal server)") - publicBinding = flag.String("public", "", "public http server binding [address]:port[/path], (default no public server)") + publicBinding = flag.String("public", "", "public http server binding [address]:port[/path] (default no public server)") - certFiles = flag.String("certfile", "", "to enable SSL specify path to certificate files without extension, expecting .crt and .key, (default no SSL)") + certFiles = flag.String("certfile", "", "to enable SSL specify path to certificate files without extension, expecting .crt and .key (default no SSL)") explorerURL = flag.String("explorer", "", "address of blockchain explorer") @@ -84,10 +86,11 @@ var ( chain bchain.BlockChain index *db.RocksDB txCache *db.TxCache + metrics *common.Metrics syncWorker *db.SyncWorker internalState *common.InternalState - callbacksOnNewBlockHash []func(hash string) - callbacksOnNewTxAddr []func(txid string, addr string) + callbacksOnNewBlock []bchain.OnNewBlockFunc + callbacksOnNewTxAddr []bchain.OnNewTxAddrFunc chanOsSignal chan os.Signal inShutdown int32 ) @@ -154,9 +157,9 @@ func main() { glog.Fatal("config: ", err) } - gspt.SetProcTitle("blockbook-" + normalizeName(coin)) + // gspt.SetProcTitle("blockbook-" + normalizeName(coin)) - metrics, err := common.GetMetrics(coin) + metrics, err = common.GetMetrics(coin) if err != nil { glog.Fatal("metrics: ", err) } @@ -165,7 +168,7 @@ func main() { glog.Fatal("rpc: ", err) } - index, err = db.NewRocksDB(*dbPath, *dbCache, chain.GetChainParser(), metrics) + index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics) if err != nil { glog.Fatal("rocksDB: ", err) } @@ -239,6 +242,11 @@ func main() { return } + // report BlockbookAppInfo metric, only log possible error + if err = blockbookAppInfoMetric(index, chain, txCache, internalState, metrics); err != nil { + glog.Error("blockbookAppInfoMetric ", err) + } + var internalServer *server.InternalServer if *internalBinding != "" { internalServer, err = server.NewInternalServer(*internalBinding, *certFiles, index, chain, txCache, internalState) @@ -259,19 +267,9 @@ func main() { }() } - if *synchronize { - if err := syncWorker.ResyncIndex(nil, true); err != nil { - glog.Error("resyncIndex ", err) - return - } - if _, err = chain.ResyncMempool(nil); err != nil { - glog.Error("resyncMempool ", err) - return - } - } - var publicServer *server.PublicServer if *publicBinding != "" { + // start public server in limited functionality, extend it after sync is finished by calling ConnectFullPublicInterface publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, txCache, *explorerURL, metrics, internalState, *debugMode) if err != nil { glog.Error("socketio: ", err) @@ -288,15 +286,32 @@ func main() { } } }() - callbacksOnNewBlockHash = append(callbacksOnNewBlockHash, publicServer.OnNewBlockHash) + callbacksOnNewBlock = append(callbacksOnNewBlock, publicServer.OnNewBlock) callbacksOnNewTxAddr = append(callbacksOnNewTxAddr, publicServer.OnNewTxAddr) } if *synchronize { - // start the synchronization loops after the server interfaces are started + internalState.SyncMode = true + internalState.InitialSync = true + if err := syncWorker.ResyncIndex(nil, true); err != nil { + glog.Error("resyncIndex ", err) + return + } + var mempoolCount int + if mempoolCount, err = chain.ResyncMempool(nil); err != nil { + glog.Error("resyncMempool ", err) + return + } + internalState.FinishedMempoolSync(mempoolCount) go syncIndexLoop() go syncMempoolLoop() - go storeInternalStateLoop() + internalState.InitialSync = false + } + go storeInternalStateLoop() + + if *publicBinding != "" { + // start full public interface + publicServer.ConnectFullPublicInterface() } if *blockFrom >= 0 { @@ -334,6 +349,25 @@ func main() { } } +func blockbookAppInfoMetric(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState, metrics *common.Metrics) error { + api, err := api.NewWorker(db, chain, txCache, is) + if err != nil { + return err + } + si, err := api.GetSystemInfo(false) + if err != nil { + return err + } + metrics.BlockbookAppInfo.Reset() + metrics.BlockbookAppInfo.With(common.Labels{ + "blockbook_version": si.Blockbook.Version, + "blockbook_commit": si.Blockbook.GitCommit, + "backend_version": si.Backend.Version, + "backend_subversion": si.Backend.Subversion, + "backend_protocol_version": si.Backend.ProtocolVersion}).Set(float64(0)) + return nil +} + func newInternalState(coin string, coinShortcut string, d *db.RocksDB) (*common.InternalState, error) { is, err := d.LoadInternalState(coin) if err != nil { @@ -399,9 +433,9 @@ func syncIndexLoop() { glog.Info("syncIndexLoop stopped") } -func onNewBlockHash(hash string) { - for _, c := range callbacksOnNewBlockHash { - c(hash) +func onNewBlockHash(hash string, height uint32) { + for _, c := range callbacksOnNewBlock { + c(hash, height) } } @@ -431,8 +465,8 @@ func storeInternalStateLoop() { lastCompute := time.Now() // randomize the duration between ComputeInternalStateColumnStats to avoid peaks after reboot of machine with multiple blockbooks computePeriod := 9*time.Hour + time.Duration(rand.Float64()*float64((2*time.Hour).Nanoseconds())) - lastLogMemory := time.Now() - logMemoryPeriod := 15 * time.Minute + lastAppInfo := time.Now() + logAppInfoPeriod := 15 * time.Minute glog.Info("storeInternalStateLoop starting with db stats recompute period ", computePeriod) tickAndDebounce(storeInternalStatePeriodMs*time.Millisecond, (storeInternalStatePeriodMs-1)*time.Millisecond, chanStoreInternalState, func() { if !computeRunning && lastCompute.Add(computePeriod).Before(time.Now()) { @@ -449,17 +483,20 @@ func storeInternalStateLoop() { if err := index.StoreInternalState(internalState); err != nil { glog.Error("storeInternalStateLoop ", errors.ErrorStack(err)) } - if lastLogMemory.Add(logMemoryPeriod).Before(time.Now()) { + if lastAppInfo.Add(logAppInfoPeriod).Before(time.Now()) { glog.Info(index.GetMemoryStats()) - lastLogMemory = time.Now() + if err := blockbookAppInfoMetric(index, chain, txCache, internalState, metrics); err != nil { + glog.Error("blockbookAppInfoMetric ", err) + } + lastAppInfo = time.Now() } }) glog.Info("storeInternalStateLoop stopped") } -func onNewTxAddr(txid string, addr string) { +func onNewTxAddr(txid string, addr string, isOutput bool) { for _, c := range callbacksOnNewTxAddr { - c(txid, addr) + c(txid, addr, isOutput) } } diff --git a/build/docker/deb/gpg-keys/bcash-shammah.chancellor.asc b/build/docker/deb/gpg-keys/bcash-shammah.chancellor.asc new file mode 100644 index 00000000..9f7d73d2 --- /dev/null +++ b/build/docker/deb/gpg-keys/bcash-shammah.chancellor.asc @@ -0,0 +1,98 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFpZENYBEACgAfjxYOe3IG0/IUmrpTExiWFvBsVAI2qDn5F11vuimbr1YMGX +PqcmFGWrYZM/zEQCO6xNZEIQ9rh79N15hXfLc9dlLx8PjuxPHNKnPDprpdsYpa37 +Dw0ufHpv+av2qqwG7o9zUan2a+QqsWXjCWhdJVNwYOFY9uRBKqkYMf+r0VOFGPIX +Pjb0n5TXdpI0FFvydeFb+HABnYqbrPcEAQQETpVukQZQtmE8UBDueO+Hz0RvJAw+ +xz79ks4muyPf4MGoKOu6xWM2mViXeP1AN+Mi3nV3EUVPKLxS2EyByrgWEh8iIdgj +vu2fRSaHSuhJWbVSkMnAPJsC3bQE/aZZ8PtaLvgVUH/0euVoKHmxIg8bj1RzNvYb +VZ2VL8Sp8xHIptkTwhgj+spi88BDI1xkBXaHt0ZlddyhgMtK7dl75mn6d+UsGQsT +4dovbtRWRFU1hww7utgtgJniwOYMzhLdY6+OZ8CSA/uB1bTGRigbSd3UtTEwHnQi +xEkHzuF4IoIbaIRLluCASs331vwk+cCZoXD8hdNQdVOwI/4Ss16RAdMFa+DGbBKb +OUpnHBx9wmGH/nq1rfBZD1r/sE7X3LhDeQfqHrRGZOFTAF3CBsiZr7SDAJMCPys6 +7y4KQtC+Kfyku+iDNsnObbCXc86qIrYEOspwMqrZgT3FmA0H+cX9EtoE8QARAQAB +tEhTaGFtbWFoIENoYW5jZWxsb3IgKEJpdGNvaW4gQUJDIGtleSkgPHNoYW1tYWgu +Y2hhbmNlbGxvckBiaXRjb2luYWJjLm9yZz6JAhwEEAEIAAYFAlsWzFAACgkQjVUt +ITLQxfZMCBAAmWotQ7ow3WIEdnBYUxZMMdGxU0WRz13lN+4YnwxAeKoBcpNwFhSz +0wGZAAVzm/hH2aLNMY/Xqt4vq2vuHC1ovOrRIL7Wk0msT/enlyPmK1j+r2zNz1Qk +92ybAfuZ2r/8zO6M1NXOrhFEOmBn6lPLQu79P6T0wGfQue3O0XPx53+WYKmNoyq7 +DQRphmBbRJikdz5TP1ffEUZ+AL901Q/b3gJ4fLpowaJ44C4J+cH++OoLY2lQFPVR +v2JyVa8SeUFfB19ki0m0r9gYEm7RDHlwcLPKwuM45armV6JTE+PW31XZqsE3Xw5h +vGGOT9bY+KoMfgplYJYr9XH1qNgUBYJZ91BAs0vuqieIoMsMHl5h+ii/MDz6eJ9y +5mwR1BKuhS0oIgvoviVU/GhEpo0GQqhsaOejnNl1/kYFpHzosFSf+9EmLWHgicKO +zwumIwsi+AjSI97pg8K8YaxkaA/uagHkutQ1oFgt/tAd6eOGnyNDCBBRSbYVuFJV +YoiRG8IaIHZkUGmrHqDeZd2Qk5YwIG7tLAX9v0cgMlDLFLx1B8LQSY9wH06zjV1a +mY69q3RzQ3mvKCf2QidB5o1FA/CV9Sn0nTvN0UKyYIXNr/KeoyCZkBuge+hD5au/ +OBrtCjGTzq6496hJbwQR83O6S3r4t4KBz2isQuoksrdxwR0cHg65hBqJAhwEEAEI +AAYFAls6O2kACgkQ5EhjVueoHSwGlA/6ApGOSYwQlppEgmwrN2BZLEmfN8pfn4ES +ymYoFKn5/4R2A11Xp5w2z9EE9JW7/yLnoar0jR/Hw16TPceiVoyRD1r5MCEeMDmj +TymDJF4p5ZeiWqkFq1CAvEdCsNB952O2F5PmqGnIm0SW7H8h49Hy1c1Cpa+d0xgg +eQkoEqtihxWabMJ7UW1r6T77dEDoPyAcP2Yi8Yv/e0Eaayv58gxB7ieGi+NsAXLL +LiGkRXhP+FHpBS9yLnxqLEPcDfivMaL3CGocJziA9GlZgSQGUjBkaVoI+Ynd9JTw +C30XususiGthXrqF5e56PIVubOhrxqAlKoxahx/kedRfWPlD/MApNxaokgmDqBpi +E2YLPzaY/RKek/AC5UAgw97yiDactt+UICcHbtG/tznQ6ECO8llI7+DSAWAKCNi1 +iCPZmdjGqv+j/S5xR3X3EdB0LoXpqE6HJE2E1+mUN/QCsz/6u/zmw1Zj5x9niPOp +g5PiIPDTLrhropMKMmvxdfKfKWOXD+I9b/tKldWi7Dz8MPlvXpkEqeA8HKRK8Ek7 +bq9iLdqGFTP/0T+nU77baOmMMSHsoimIIZRysg04n8WR6A4AAOACX/BoB6agxLn8 +1N//xMTqUA0Aror9dLNXl0iCbUshnFkx1wiq93rdZAn6nCI0wlFtg2yhh/dWo1sZ +caGNIgGdxn2JAhwEEAEIAAYFAls7ja0ACgkQXXkiu9ZJxKdf1RAArXapCE/zxWpd +f9cNOClRBLXZlj7FiW7Mf63DgdO0ae4gK4sL812bSwEH4KeuQxkugYsOcH6oezWh +mXxJ8lHOeCFbW+a6VBx7kclUweEOpFtcfrJU1bgLywGM6FtWNn1m1mAMKCrqDgPQ +U2L6UnhY+H3QgRqqYo1nzRsfpqnAxw+7nsquD4ziQS/Mumbj2C0Gjg5ScjkfAidk +okp5jdoIEbanjKQ8zuSRWvomGeT6UF1AmPA3bNtN3ROD9co4H8fKeFKamrCzVZul ++eu5/Y+cCvQ5PEHZoflZYwa3qdTgQJhIlzBmGIipYstHVPfyeFKN9JYXv01FNvDT +h1ebvDzgvaBB/lln3CxWHSEkv7jTu902ljUO9iuPdd0tX89FN1FlLCqeXXoT9Qfi +NOaRJzNDkqdHl53VrhuNH5mZWXXAuD5hkZo1I5A90YuOLOQ5wnl/8MbDS4VW8tpK +HWqn5b0GFacB2xsSphYpletkQnwQWFy352NBveDf68E9rGA9WS5B7cVvnSiNZWNT +5hzE5krS6VZjSIo062PVySTWAqgCy1OWdEPZXo71BpKHV3AgBTAaoPj6EWDFaqS3 +4/5s4odJBUgtceEg4naXFp5GdrrixzInLwjNu1FdpU3NM1thelWzErYi8k8OMVKy +QroQJ+ObZ3PIKBaLgAyOTJq6TZqxa/2JAjcEEwEIACsFAlpZENYJEH05WMREJ2dK +AhsDBQkHhM4ABQsJCAcCBhUKCQgLAgQWAgMBAABVdw/+Jc4K7/8rcWawTG7u2jQG +C3un44dHrVwFxft1IQWbc1ZYx7xvTFtAyOwatwL+PnHjJlMK6ODztJnNHS95CqPS +9dkUztrlQF+j4gmc6/7h0Ew3M56N9tVOXP/3Un+kPgGLdWvVYf0YZsy0r8BO7vQb +PXk2coKiWwfony55DDt9RaxLNlWukKq3dg+vejZzzucfKuZIlUcQDRJO4rCXxMBv +Oo/BtgfOXACjzi4O8aF2TNzEKyXLp8KqcB8j2knu2PIJidr8fwzyBwe3yF8tox3x +4nvSn5IYvvvP4EG8fhsGVBEc+EwZJnbrk5afRBnM/uYW+e+/7vBQwlZyvFOr/zMJ +rhbtcWwW8VkFhvbwc40K7qsiJOxfzXMfGVPtutbzhz8sRivija03LrE0ysgesoM4 +LpxsHDhwScwTTupZ8qsvoNDqgyuJvEUeNXKv1fdY9IVtKgJuU5JHcLOyhUB8/xkd +2qle6fKQK1k776HhdmseU6Ba3cbe3zZYJfYGnoHXttfOZ5QhKNlFhyCdmv7CImP3 +KFYKfYcbx+CVhGJpm5YEYk3WW2FyuBx80WrEbztGcvqZzPQmNPAhvRkVErNoxA/A +KCuLTNrz0jivgBny6JKGdY8drXRafovBroJcLGGFfTVjgZouUis1VlZX4Bo4Twcq +Pc5AK4OXCa/WK1IjXTSblDWJAlQEEwEIAD4WIQR6VaRPOjI5gnyKWU59OVjERCdn +SgUCWlkQ1gIbAwUJB4TOAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRB9OVjE +RCdnSmNWD/47fhdIy2gdHKl+BqjUezcmBs5eR5V8/pz6kbT1+cDFHZ+ImcK+jf8I +8oQPeFWecBF1ZERoLaIRD6UwIVs6z7yAGYlCeweitWf7cfgvIvzP9b/sLweAfwD8 +fNfKTQuw3wx6Yr66+y42lM5dasu+WyzBYLQhQI6q+WgsRL5j7IuNFTdtAc/gNY5c +GlEPXMl9RCjJlCeyo9UNvMeL61UrkPUMVOOqH4Bl8yatPilu0vyGtYogp+XTbmEk +ZJ1BWhL4Y8VkVdBhX9LhYZ+klI6UjNxnZEnVwTwVhZCMXBgi6t4X8EJqeVa5/gb5 +LQ7L261z3y3wzyKZ9+wTjjihXaHX+5n7b1oTZF08+1cY9qe1tDIWfsDY5HjoJb15 +MSqy5EdTJsH+h2JlzMD3t57pTCmGXPmA8mdec78q5DA2PlCdc0qewOTDJ8d56J/x +4PrSZ8TZDRIKA3fflRfWwKrAmB+xwR4cvAXbNC1PxC4BvlVOFKBseDHv7VwWJvnH +0Gnc6hdHdzJJYtTQBglOH6CifHBnksc6/pnFedQ8yWRKXFY3rsaqpzkiiqCPl9jG +QCPWAEvehT6v4iFFiQkZ9OTjBVmT9Ea6BvhpCplpgPo+AFKS3L1mMZYf8RIHazdo +fQof6w/oMxmwQWT/k3O0Z4JRNW0p2eyB8RS3tK3YT2zY3y4TSyYqY7kCDQRaWRDW +ARAA0zg0VamqvOmzSg+viNRFwPmetdPNmiKq7feDail0r8vtfAIKuXcjR6EF0FEB +w9eWlzbX0Eu6c/MVPRycj9Ka+7KqZbEAlOOC97gzA2tcRar8aEM1ZXDBzyr3ZbbK +aH6/XPYm+oxnYxwlMY99/LpiTcOkQyc4yJCXSC8GV9frQhVOGPI7gPca7DjroGdb +as/uTiyPNp4xq0JTI/xWYLRtH+dTEfH2/vTpoXku0itdfR4uoCPFzOYCegfObIOX +3UsUvkoNKWBoQGc14YarTMAjagDDUBhnhJ84b/ftOWTvyYyq/QR4oJdGUkTp854Z +4r2J8cWzW2tpuALxUrHs4AcTmW3cD37B7j3lhzpfA3NGLYwDBQfQkFMwtRIxskf3 +EG7Ednq5fXJWCekIYSji8bDaGu/A5exuj5H0Mg3DKUcVQ+p2bRh6fjKZwG6gEQWU +rxu6RsjFYg6KuLUb0q0Y0Vt/vvLASkY4FoIxjj0zGR+KBlg8yJsU44HdiI40LRPy +2juYgGAhL2j53Yrr+ID0ZFbe9TpQePcBfD5lEWxVH9aBjjfEz1TUYguNQXo3mVrO +OWZek6f9Uh5Yt+WSPSrZfzbRsROcv3OOC83lSYcL2ugiFjt9oGXZBYhfYNLCKDpW +10nGVe0oVSjIA53QIdhQ233WPvwi6ur5Fa8j5OvaIWnwpecAEQEAAYkCPAQYAQgA +JhYhBHpVpE86MjmCfIpZTn05WMREJ2dKBQJaWRDWAhsMBQkHhM4AAAoJEH05WMRE +J2dK5VMP/2IIPi707oF3/iflM3JvQukjmGNLpm3sVrx0baSODW2IfSNQeMWYXvoa +xgaKrMG+WGzr1P/D1jv+HD8gxjakTvZYooJLHF/9nmZE1bD/p0BGFpoRuvX9L/J9 +Yqu32S89lYqnvyXY3c6Aqt/sJj1SHCPdFWMT/kcYjGHeBaX2Ub5+zVm4THESwssv +GsT37N6Wzz0P2wI1rSRu29evKbRTebRIZXaSWiVlx4P7Y6CNFNptzkH7oqf6JmkB +qaQZuOZ3dhe6Z9HetOFN9D5X0ju3DgUITEmF6BYekGBEv/HVJs6rbTb9us8sV4Vu +WvmTYCCIn/fOnmFD6WomjVeX1E6DJ9YvRtCPtau0JcxxBy1L3/3IS2fEf3zpTK6O +aX2GnpijmLFweDGD7CocaEmfw3xUvwgDEpIIvaMNnlciujZFd+8MMJ/JeQ+l5xuw +yWcUciwnWIoCQBOTl3u0u/mWeQsHQr81pirknugjJ8OCPyMUbtsNm9aAySFj6lra +O18EF0uVo+oYDulM+2InSjEJQLYj392XOG4sWkO0bPWJ6FG8VFC2MH09bnsU7dyH +mN6ZWn6JZI3uPY3UbRsXTnVws4OG4bBCpuCe0mJ35gWGAqj0ZDm8SA5Utx4j2oEl +h0HChP/lLDxJ5t5PfBVyudK+xewFFx/cX+4f/n7GZp2+wvduWrs7 +=Grfn +-----END PGP PUBLIC KEY BLOCK----- diff --git a/common/internalstate.go b/common/internalstate.go index 1053649c..9479d057 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -37,6 +37,10 @@ type InternalState struct { LastStore time.Time `json:"lastStore"` + // true if application is with flag --sync + SyncMode bool `json:"syncMode"` + + InitialSync bool `json:"initialSync"` IsSynchronized bool `json:"isSynchronized"` BestHeight uint32 `json:"bestHeight"` LastSync time.Time `json:"lastSync"` @@ -64,6 +68,14 @@ func (is *InternalState) FinishedSync(bestHeight uint32) { is.LastSync = time.Now() } +// UpdateBestHeight sets new best height, without changing IsSynchronized flag +func (is *InternalState) UpdateBestHeight(bestHeight uint32) { + is.mux.Lock() + defer is.mux.Unlock() + is.BestHeight = bestHeight + is.LastSync = time.Now() +} + // FinishedSyncNoChange marks end of synchronization in case no index update was necessary, it does not update lastSync time func (is *InternalState) FinishedSyncNoChange() { is.mux.Lock() diff --git a/common/metrics.go b/common/metrics.go index e17cd489..9451180b 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -21,6 +21,7 @@ type Metrics struct { MempoolSize prometheus.Gauge DbColumnRows *prometheus.GaugeVec DbColumnSize *prometheus.GaugeVec + BlockbookAppInfo *prometheus.GaugeVec } type Labels = prometheus.Labels @@ -139,6 +140,14 @@ func GetMetrics(coin string) (*Metrics, error) { }, []string{"column"}, ) + metrics.BlockbookAppInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "blockbook_app_info", + Help: "Information about blockbook and backend application versions", + ConstLabels: Labels{"coin": coin}, + }, + []string{"blockbook_version", "blockbook_commit", "backend_version", "backend_subversion", "backend_protocol_version"}, + ) v := reflect.ValueOf(metrics) for i := 0; i < v.NumField(); i++ { diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 78c74336..c0beb4bd 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.17.1", - "binary_url": "https://download.bitcoinabc.org/0.17.1/linux/bitcoin-abc-0.17.1-x86_64-linux-gnu.tar.gz", + "version": "0.18.2", + "binary_url": "https://download.bitcoinabc.org/0.18.2/linux/bitcoin-abc-0.18.2-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "eccf8b61ba0549f6839e586c7dc6fc4bf6d7591ac432aaea8a7df0266b113d27", + "verification_source": "28d8511789a126aff16e256a03288948f2660c3c8cb0a4c809c5a8618a519a16", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -44,7 +44,7 @@ "system_user": "blockbook-bcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://bitcoincash.blockexplorer.com", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index ad6bda08..3d0e4892 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash-testnet", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.17.1", - "binary_url": "https://download.bitcoinabc.org/0.17.1/linux/bitcoin-abc-0.17.1-x86_64-linux-gnu.tar.gz", + "version": "0.18.2", + "binary_url": "https://download.bitcoinabc.org/0.18.2/linux/bitcoin-abc-0.18.2-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "eccf8b61ba0549f6839e586c7dc6fc4bf6d7591ac432aaea8a7df0266b113d27", + "verification_source": "28d8511789a126aff16e256a03288948f2660c3c8cb0a4c809c5a8618a519a16", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -44,7 +44,7 @@ "system_user": "blockbook-bcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://bitcoincash.blockexplorer.com", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/bgold.json b/configs/coins/bgold.json index 3eb019a6..cc14bdf2 100644 --- a/configs/coins/bgold.json +++ b/configs/coins/bgold.json @@ -22,10 +22,10 @@ "package_name": "backend-bgold", "package_revision": "satoshilabs-1", "system_user": "bgold", - "version": "0.15.1", - "binary_url": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.1/bitcoin-gold-0.15.1-x86_64-linux-gnu.tar.gz", + "version": "0.15.2", + "binary_url": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.2/bitcoin-gold-0.15.2-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.1/SHA256SUMS.asc", + "verification_source": "https://github.com/BTCGPU/BTCGPU/releases/download/v0.15.2/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -239,11 +239,11 @@ "system_user": "blockbook-bgold", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://btg-explorer.trezor.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Bitcoin Gold:0.15.1/", + "subversion": "/Bitcoin Gold:0.15.2/", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index ec93e446..97798bd8 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -22,10 +22,10 @@ "package_name": "backend-bitcoin", "package_revision": "satoshilabs-1", "system_user": "bitcoin", - "version": "0.16.1", - "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.1/bitcoin-0.16.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.1/SHA256SUMS.asc", + "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "/explorer", + "explorer_url": "", "additional_params": "-dbcache=1073741824", "block_chain": { "parse": true, diff --git a/configs/coins/bitcoin_testnet.json b/configs/coins/bitcoin_testnet.json index 760163be..a2ef7e0d 100644 --- a/configs/coins/bitcoin_testnet.json +++ b/configs/coins/bitcoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bitcoin-testnet", "package_revision": "satoshilabs-1", "system_user": "bitcoin", - "version": "0.16.1", - "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.1/bitcoin-0.16.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.1/SHA256SUMS.asc", + "verification_source": "https://bitcoin.org/bin/bitcoin-core-0.16.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-bitcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "/explorer", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/dash.json b/configs/coins/dash.json index d2895671..e9156efb 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -22,10 +22,10 @@ "package_name": "backend-dash", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.12.3", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/dashcore-0.12.3.2-x86_64-linux-gnu.tar.gz", + "version": "0.12.3.3", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/dashcore-0.12.3.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" @@ -47,11 +47,11 @@ "system_user": "blockbook-dash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://dash-explorer.trezor.io", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Dash Core:0.12.3.2/", + "subversion": "/Dash Core:0.12.3.3/", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index 38407e37..e8ebaaa1 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-dash-testnet", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.12.3", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/dashcore-0.12.3.2-x86_64-linux-gnu.tar.gz", + "version": "0.12.3.3", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/dashcore-0.12.3.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.2/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.12.3.3/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" @@ -47,11 +47,11 @@ "system_user": "blockbook-dash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://dash-explorer.trezor.io", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Dash Core:0.12.3.2/", + "subversion": "/Dash Core:0.12.3.3/", "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, diff --git a/configs/coins/ethereum-classic.json b/configs/coins/ethereum-classic.json index e800da46..f7e49b56 100644 --- a/configs/coins/ethereum-classic.json +++ b/configs/coins/ethereum-classic.json @@ -40,7 +40,7 @@ "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", "explorer_url": "https://gastracker.io/", - "additional_params": "", + "additional_params": "-resyncindexperiod=4441", "block_chain": { "parse": true, "mempool_workers": 8, diff --git a/configs/coins/ethereum.json b/configs/coins/ethereum.json index fcc63d3d..5423c5ed 100644 --- a/configs/coins/ethereum.json +++ b/configs/coins/ethereum.json @@ -21,10 +21,10 @@ "package_name": "backend-ethereum", "package_revision": "satoshilabs-1", "system_user": "ethereum", - "version": "1.8.10-eae63c51", - "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.10-eae63c51.tar.gz", + "version": "1.8.15-89451f7c", + "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.15-89451f7c.tar.gz", "verification_type": "gpg", - "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.10-eae63c51.tar.gz.asc", + "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.15-89451f7c.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/geth --ipcdisable --syncmode full --cache 1024 --nat none --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port 38336 --ws --wsaddr 0.0.0.0 --wsport {{.Ports.BackendRPC}} --wsorigins \"*\" --rpc --rpcport 8136 -rpcaddr 0.0.0.0 --rpccorsdomain \"*\" --rpcvhosts \"*\" 2>>{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", diff --git a/configs/coins/ethereum_testnet_ropsten.json b/configs/coins/ethereum_testnet_ropsten.json index a236e16d..42fe3fd0 100644 --- a/configs/coins/ethereum_testnet_ropsten.json +++ b/configs/coins/ethereum_testnet_ropsten.json @@ -20,13 +20,13 @@ "package_name": "backend-ethereum-testnet-ropsten", "package_revision": "satoshilabs-1", "system_user": "ethereum", - "version": "1.8.10-eae63c51", - "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.10-eae63c51.tar.gz", + "version": "1.8.15-89451f7c", + "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.15-89451f7c.tar.gz", "verification_type": "gpg", - "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.10-eae63c51.tar.gz.asc", + "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.15-89451f7c.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], - "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/geth --testnet --ipcdisable --cache 1024 --nat none --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port 48336 --ws --wsaddr 0.0.0.0 --wsport {{.Ports.BackendRPC}} --wsorigins \"*\" 2>>{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", + "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/geth --testnet --syncmode full --ipcdisable --cache 1024 --nat none --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port 48336 --ws --wsaddr 0.0.0.0 --wsport {{.Ports.BackendRPC}} --wsorigins \"*\" 2>>{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log", "postinst_script_template": "", "service_type": "simple", diff --git a/configs/coins/litecoin.json b/configs/coins/litecoin.json index e4541bf9..3f9747fd 100644 --- a/configs/coins/litecoin.json +++ b/configs/coins/litecoin.json @@ -22,10 +22,10 @@ "package_name": "backend-litecoin", "package_revision": "satoshilabs-1", "system_user": "litecoin", - "version": "0.16.0", - "binary_url": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-linux-signatures.asc", + "verification_source": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-linux-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/litecoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-litecoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://ltc-explorer.trezor.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/litecoin_testnet.json b/configs/coins/litecoin_testnet.json index 819ae8c0..8eae9a61 100644 --- a/configs/coins/litecoin_testnet.json +++ b/configs/coins/litecoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-litecoin-testnet", "package_revision": "satoshilabs-1", "system_user": "litecoin", - "version": "0.16.0", - "binary_url": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://download.litecoin.org/litecoin-0.16.0/linux/litecoin-0.16.0-linux-signatures.asc", + "verification_source": "https://download.litecoin.org/litecoin-0.16.3/linux/litecoin-0.16.3-linux-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/litecoin-qt" @@ -47,7 +47,7 @@ "system_user": "blockbook-litecoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://ltc-explorer.trezor.io", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/monacoin.json b/configs/coins/monacoin.json index 563b1db6..5bf11b38 100644 --- a/configs/coins/monacoin.json +++ b/configs/coins/monacoin.json @@ -22,10 +22,10 @@ "package_name": "backend-monacoin", "package_revision": "satoshilabs-1", "system_user": "monacoin", - "version": "0.15.1", - "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-signatures.asc", + "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/monacoin-qt" diff --git a/configs/coins/monacoin_testnet.json b/configs/coins/monacoin_testnet.json index 9e0d06ab..23ac8ce6 100644 --- a/configs/coins/monacoin_testnet.json +++ b/configs/coins/monacoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-monacoin-testnet", "package_revision": "satoshilabs-1", "system_user": "monacoin", - "version": "0.15.1", - "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.15.1/monacoin-0.15.1-signatures.asc", + "verification_source": "https://github.com/monacoinproject/monacoin/releases/download/monacoin-0.16.3/monacoin-0.16.3-signatures.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/monacoin-qt" diff --git a/configs/coins/namecoin.json b/configs/coins/namecoin.json index 754ee628..4cb9933f 100644 --- a/configs/coins/namecoin.json +++ b/configs/coins/namecoin.json @@ -22,10 +22,10 @@ "package_name": "backend-namecoin", "package_revision": "satoshilabs-1", "system_user": "namecoin", - "version": "0.13.99", - "binary_url": "https://namecoin.org/files/namecoin-core-0.13.99-name-tab-beta1-notreproduced/namecoin-0.13.99-x86_64-linux-gnu.tar.gz", + "version": "0.16.3", + "binary_url": "https://www.namecoin.org/files/namecoin-core-0.16.3/namecoin-0.16.3-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "294b1106001d6ea2b9d9ee6a655021ef207a24e8f1dec8efd5899728b3849129", + "verification_source": "14ebaaf6f22f69b057a5bcb9b6959548f0a3f1b62cc113f19581d2297044827e", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/namecoin-qt" @@ -54,7 +54,7 @@ "system_user": "blockbook-namecoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://namecha.in/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/vertcoin.json b/configs/coins/vertcoin.json index d68a0a0f..fa14c06a 100644 --- a/configs/coins/vertcoin.json +++ b/configs/coins/vertcoin.json @@ -22,10 +22,10 @@ "package_name": "backend-vertcoin", "package_revision": "satoshilabs-1", "system_user": "vertcoin", - "version": "0.13.2", - "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip", + "version": "0.13.3", + "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip", "verification_type": "gpg", - "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip.sig", + "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip.sig", "extract_command": "unzip -d backend", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/vertcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -45,7 +45,7 @@ "system_user": "blockbook-vertcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://insight.vertcoin.org", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/vertcoin_testnet.json b/configs/coins/vertcoin_testnet.json index 314929c3..2f8dfaee 100644 --- a/configs/coins/vertcoin_testnet.json +++ b/configs/coins/vertcoin_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-vertcoin-testnet", "package_revision": "satoshilabs-1", "system_user": "vertcoin", - "version": "0.13.2", - "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip", + "version": "0.13.3", + "binary_url": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip", "verification_type": "gpg", - "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.2/vertcoind-v0.13.2-linux-amd64.zip.sig", + "verification_source": "https://github.com/vertcoin-project/vertcoin-core/releases/download/0.13.3/vertcoind-v0.13.3-linux-amd64.zip.sig", "extract_command": "unzip -d backend", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/vertcoind -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -45,7 +45,7 @@ "system_user": "blockbook-vertcoin", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://insight.vertcoin.org/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index ab65b246..d55165d8 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "1.1.1", - "binary_url": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz", + "version": "2.0.0", + "binary_url": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz", "verification_type": "gpg", - "verification_source": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz.asc", + "verification_source": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -47,7 +47,7 @@ "system_user": "blockbook-zcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://zcash.blockexplorer.com/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index c2793c96..5f6d2ca4 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash-testnet", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "1.1.1", - "binary_url": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz", + "version": "2.0.0", + "binary_url": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz", "verification_type": "gpg", - "verification_source": "https://z.cash/downloads/zcash-1.1.1-linux64.tar.gz.asc", + "verification_source": "https://z.cash/downloads/zcash-2.0.0-linux64.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -47,7 +47,7 @@ "system_user": "blockbook-zcash", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://explorer.testnet.z.cash/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/environ.json b/configs/environ.json index 1f293514..3adf012c 100644 --- a/configs/environ.json +++ b/configs/environ.json @@ -1,5 +1,5 @@ { - "version": "0.0.6", + "version": "0.1.0", "backend_install_path": "/opt/coins/nodes", "backend_data_path": "/opt/coins/data", "blockbook_install_path": "/opt/coins/blockbook", diff --git a/db/bulkconnect.go b/db/bulkconnect.go index eeefd234..60ca7a64 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -15,7 +15,6 @@ import ( // 2) rocksdb seems to handle better fewer larger batches than continuous stream of smaller batches type bulkAddresses struct { - height uint32 bi BlockInfo addresses map[string][]outpoint } @@ -154,10 +153,10 @@ func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) { func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { for _, ba := range b.bulkAddresses { - if err := b.d.storeAddresses(wb, ba.height, ba.addresses); err != nil { + if err := b.d.storeAddresses(wb, ba.bi.Height, ba.addresses); err != nil { return err } - if err := b.d.writeHeight(wb, ba.height, &ba.bi, opInsert); err != nil { + if err := b.d.writeHeight(wb, ba.bi.Height, &ba.bi, opInsert); err != nil { return err } } @@ -190,12 +189,12 @@ func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) erro } } b.bulkAddresses = append(b.bulkAddresses, bulkAddresses{ - height: block.Height, bi: BlockInfo{ - Hash: block.Hash, - Time: block.Time, - Txs: uint32(len(block.Txs)), - Size: uint32(block.Size), + Hash: block.Hash, + Time: block.Time, + Txs: uint32(len(block.Txs)), + Size: uint32(block.Size), + Height: block.Height, }, addresses: addresses, }) diff --git a/db/dboptions.go b/db/dboptions.go index 2e0da942..f982f8a3 100644 --- a/db/dboptions.go +++ b/db/dboptions.go @@ -38,7 +38,7 @@ func boolToChar(b bool) C.uchar { } */ -func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options { +func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache, maxOpenFiles int) *gorocksdb.Options { blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions() blockOpts.SetBlockSize(32 << 10) // 32kB blockOpts.SetBlockCache(c) @@ -54,7 +54,7 @@ func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache) *gorocksdb.Options opts.SetBytesPerSync(8 << 20) // 8MB opts.SetWriteBufferSize(1 << 27) // 128MB opts.SetMaxBytesForLevelBase(1 << 27) // 128MB - opts.SetMaxOpenFiles(25000) + opts.SetMaxOpenFiles(maxOpenFiles) opts.SetCompression(gorocksdb.LZ4HCCompression) return opts } diff --git a/db/rocksdb.go b/db/rocksdb.go index b16647cd..db46b83c 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -33,17 +33,29 @@ func RepairRocksDB(name string) error { return gorocksdb.RepairDb(name, opts) } +type connectBlockStats struct { + txAddressesHit int + txAddressesMiss int + balancesHit int + balancesMiss int +} + // RocksDB handle type RocksDB struct { - path string - db *gorocksdb.DB - wo *gorocksdb.WriteOptions - ro *gorocksdb.ReadOptions - cfh []*gorocksdb.ColumnFamilyHandle - chainParser bchain.BlockChainParser - is *common.InternalState - metrics *common.Metrics - cache *gorocksdb.Cache + path string + db *gorocksdb.DB + wo *gorocksdb.WriteOptions + ro *gorocksdb.ReadOptions + cfh []*gorocksdb.ColumnFamilyHandle + chainParser bchain.BlockChainParser + is *common.InternalState + metrics *common.Metrics + cache *gorocksdb.Cache + maxOpenFiles int + cbs connectBlockStats + chanUpdateBalance chan updateBalanceData + chanUpdateBalanceResult chan error + updateBalancesMap map[string]*AddrBalance } const ( @@ -58,12 +70,12 @@ const ( var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} -func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { +func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter - opts := createAndSetDBOptions(10, c) + opts := createAndSetDBOptions(10, c, openFiles) // opts for addresses without bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter - optsAddresses := createAndSetDBOptions(0, c) + optsAddresses := createAndSetDBOptions(0, c, openFiles) // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) @@ -75,13 +87,26 @@ func openDB(path string, c *gorocksdb.Cache) (*gorocksdb.DB, []*gorocksdb.Column // NewRocksDB opens an internal handle to RocksDB environment. Close // needs to be called to release it. -func NewRocksDB(path string, cacheSize int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { - glog.Infof("rocksdb: open %s, version %v", path, dbVersion) +func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { + glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) c := gorocksdb.NewLRUCache(cacheSize) - db, cfh, err := openDB(path, c) + db, cfh, err := openDB(path, c, maxOpenFiles) wo := gorocksdb.NewDefaultWriteOptions() ro := gorocksdb.NewDefaultReadOptions() - return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c}, nil + rdb := &RocksDB{ + path: path, + db: db, + wo: wo, + ro: ro, + cfh: cfh, + chainParser: parser, + metrics: metrics, + cache: c, + maxOpenFiles: maxOpenFiles, + cbs: connectBlockStats{}, + } + rdb.initUpdateBalancesWorker() + return rdb, nil } func (d *RocksDB) closeDB() error { @@ -119,7 +144,7 @@ func (d *RocksDB) Reopen() error { return err } d.db = nil - db, cfh, err := openDB(d.path, d.cache) + db, cfh, err := openDB(d.path, d.cache, d.maxOpenFiles) if err != nil { return err } @@ -265,6 +290,8 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { txAddressesMap := make(map[string]*TxAddresses) balances := make(map[string]*AddrBalance) if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { + // reinitialize balanceWorker so that there are no left balances in the queue + d.initUpdateBalancesWorker() return err } if err := d.storeAddresses(wb, block.Height, addresses); err != nil { @@ -347,7 +374,86 @@ func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc bchain.Address valueSat.SetInt64(0) } +func (d *RocksDB) GetAndResetConnectBlockStats() string { + s := fmt.Sprintf("%+v", d.cbs) + d.cbs = connectBlockStats{} + return s +} + +type updateBalanceData struct { + valueSat big.Int + strAddrDesc string + addrDesc bchain.AddressDescriptor + processed, output bool +} + +func (d *RocksDB) initUpdateBalancesWorker() { + if d.chanUpdateBalance != nil { + close(d.chanUpdateBalance) + } + d.chanUpdateBalance = make(chan updateBalanceData, 16) + d.chanUpdateBalanceResult = make(chan error, 16) + go d.updateBalancesWorker() +} + +// updateBalancesWorker is a single worker used to update balances in parallel to processAddressesUTXO +func (d *RocksDB) updateBalancesWorker() { + var err error + for bd := range d.chanUpdateBalance { + ab, e := d.updateBalancesMap[bd.strAddrDesc] + if !e { + ab, err = d.GetAddrDescBalance(bd.addrDesc) + if err != nil { + d.chanUpdateBalanceResult <- err + continue + } + if ab == nil { + ab = &AddrBalance{} + } + d.updateBalancesMap[bd.strAddrDesc] = ab + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ + } + // add number of trx in balance only once, address can be multiple times in tx + if !bd.processed { + ab.Txs++ + } + if bd.output { + ab.BalanceSat.Add(&ab.BalanceSat, &bd.valueSat) + } else { + ab.BalanceSat.Sub(&ab.BalanceSat, &bd.valueSat) + if ab.BalanceSat.Sign() < 0 { + d.resetValueSatToZero(&ab.BalanceSat, bd.addrDesc, "balance") + } + ab.SentSat.Add(&ab.SentSat, &bd.valueSat) + } + d.chanUpdateBalanceResult <- nil + } +} + +func (d *RocksDB) dispatchUpdateBalance(dispatchedBalances int, valueSat *big.Int, strAddrDesc string, addrDesc bchain.AddressDescriptor, processed, output bool) (int, error) { +loop: + for { + select { + // process as many results as possible + case err := <-d.chanUpdateBalanceResult: + if err != nil { + return 0, err + } + dispatchedBalances-- + // send input to be processed + case d.chanUpdateBalance <- updateBalanceData{*valueSat, strAddrDesc, addrDesc, processed, output}: + dispatchedBalances++ + break loop + } + } + return dispatchedBalances, nil +} + func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { + d.updateBalancesMap = balances + dispatchedBalances := 0 blockTxIDs := make([][]byte, len(block.Txs)) blockTxAddresses := make([]*TxAddresses, len(block.Txs)) // first process all outputs so that inputs can point to txs in this block @@ -389,22 +495,10 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string btxID: btxID, index: int32(i), }) - ab, e := balances[strAddrDesc] - if !e { - ab, err = d.GetAddrDescBalance(addrDesc) - if err != nil { - return err - } - if ab == nil { - ab = &AddrBalance{} - } - balances[strAddrDesc] = ab + dispatchedBalances, err = d.dispatchUpdateBalance(dispatchedBalances, &output.ValueSat, strAddrDesc, addrDesc, processed, true) + if err != nil { + return err } - // add number of trx in balance only once, address can be multiple times in tx - if !processed { - ab.Txs++ - } - ab.BalanceSat.Add(&ab.BalanceSat, &output.ValueSat) } } // process inputs @@ -436,6 +530,9 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string continue } txAddressesMap[stxID] = ita + d.cbs.txAddressesMiss++ + } else { + d.cbs.txAddressesHit++ } if len(ita.Outputs) <= int(input.Vout) { glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) @@ -467,26 +564,16 @@ func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string btxID: spendingTxid, index: ^int32(i), }) - ab, e := balances[strAddrDesc] - if !e { - ab, err = d.GetAddrDescBalance(ot.AddrDesc) - if err != nil { - return err - } - if ab == nil { - ab = &AddrBalance{} - } - balances[strAddrDesc] = ab + dispatchedBalances, err = d.dispatchUpdateBalance(dispatchedBalances, &ot.ValueSat, strAddrDesc, ot.AddrDesc, processed, false) + if err != nil { + return err } - // add number of trx in balance only once, address can be multiple times in tx - if !processed { - ab.Txs++ - } - ab.BalanceSat.Sub(&ab.BalanceSat, &ot.ValueSat) - if ab.BalanceSat.Sign() < 0 { - d.resetValueSatToZero(&ab.BalanceSat, ot.AddrDesc, "balance") - } - ab.SentSat.Add(&ab.SentSat, &ot.ValueSat) + } + } + for i := 0; i < dispatchedBalances; i++ { + err := <-d.chanUpdateBalanceResult + if err != nil { + return err } } return nil @@ -876,10 +963,11 @@ func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain. // BlockInfo holds information about blocks kept in column height type BlockInfo struct { - Hash string - Time int64 - Txs uint32 - Size uint32 + Hash string + Time int64 + Txs uint32 + Size uint32 + Height uint32 // Height is not packed! } func (d *RocksDB) packBlockInfo(block *BlockInfo) ([]byte, error) { @@ -959,15 +1047,21 @@ func (d *RocksDB) GetBlockInfo(height uint32) (*BlockInfo, error) { return nil, err } defer val.Free() - return d.unpackBlockInfo(val.Data()) + bi, err := d.unpackBlockInfo(val.Data()) + if err != nil { + return nil, err + } + bi.Height = height + return bi, err } func (d *RocksDB) writeHeightFromBlock(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { return d.writeHeight(wb, block.Height, &BlockInfo{ - Hash: block.Hash, - Time: block.Time, - Txs: uint32(len(block.Txs)), - Size: uint32(block.Size), + Hash: block.Hash, + Time: block.Time, + Txs: uint32(len(block.Txs)), + Size: uint32(block.Size), + Height: block.Height, }, op) } @@ -980,8 +1074,10 @@ func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, height uint32, bi *Block return err } wb.PutCF(d.cfh[cfHeight], key, val) + d.is.UpdateBestHeight(height) case opDelete: wb.DeleteCF(d.cfh[cfHeight], key) + d.is.UpdateBestHeight(height - 1) } return nil } @@ -1214,8 +1310,10 @@ func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error func dirSize(path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { - if !info.IsDir() { - size += info.Size() + if err == nil { + if !info.IsDir() { + size += info.Size() + } } return err }) @@ -1342,6 +1440,12 @@ func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, erro } } is.DbColumns = nc + // after load, reset the synchronization data + is.IsSynchronized = false + is.IsMempoolSynchronized = false + var t time.Time + is.LastMempoolSync = t + is.SyncMode = false return is, nil } diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 5b04fa3f..8e114597 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -36,7 +36,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - d, err := NewRocksDB(tmp, 100000, p, nil) + d, err := NewRocksDB(tmp, 100000, -1, p, nil) if err != nil { t.Fatal(err) } @@ -730,10 +730,11 @@ func TestRocksDB_Index_UTXO(t *testing.T) { t.Fatal(err) } iw := &BlockInfo{ - Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", - Txs: 4, - Size: 2345678, - Time: 1534859123, + Hash: "00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6", + Txs: 4, + Size: 2345678, + Time: 1534859123, + Height: 225494, } if !reflect.DeepEqual(info, iw) { t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) diff --git a/db/sync.go b/db/sync.go index 32093ab6..6bae828f 100644 --- a/db/sync.go +++ b/db/sync.go @@ -47,7 +47,7 @@ var errSynced = errors.New("synced") // ResyncIndex synchronizes index to the top of the blockchain // onNewBlock is called when new block is connected, but not in initial parallel sync -func (w *SyncWorker) ResyncIndex(onNewBlock func(hash string), initialSync bool) error { +func (w *SyncWorker) ResyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { start := time.Now() w.is.StartedSync() @@ -67,6 +67,7 @@ func (w *SyncWorker) ResyncIndex(onNewBlock func(hash string), initialSync bool) case errSynced: // this is not actually error but flag that resync wasn't necessary w.is.FinishedSyncNoChange() + w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) return nil } @@ -75,7 +76,7 @@ func (w *SyncWorker) ResyncIndex(onNewBlock func(hash string), initialSync bool) return err } -func (w *SyncWorker) resyncIndex(onNewBlock func(hash string), initialSync bool) error { +func (w *SyncWorker) resyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { remoteBestHash, err := w.chain.GetBestBlockHash() if err != nil { return err @@ -135,7 +136,7 @@ func (w *SyncWorker) resyncIndex(onNewBlock func(hash string), initialSync bool) return w.connectBlocks(onNewBlock, initialSync) } -func (w *SyncWorker) handleFork(localBestHeight uint32, localBestHash string, onNewBlock func(hash string), initialSync bool) error { +func (w *SyncWorker) handleFork(localBestHeight uint32, localBestHash string, onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { // find forked blocks, disconnect them and then synchronize again var height uint32 hashes := []string{localBestHash} @@ -163,7 +164,7 @@ func (w *SyncWorker) handleFork(localBestHeight uint32, localBestHash string, on return w.resyncIndex(onNewBlock, initialSync) } -func (w *SyncWorker) connectBlocks(onNewBlock func(hash string), initialSync bool) error { +func (w *SyncWorker) connectBlocks(onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { bch := make(chan blockResult, 8) done := make(chan struct{}) defer close(done) @@ -182,7 +183,7 @@ func (w *SyncWorker) connectBlocks(onNewBlock func(hash string), initialSync boo return err } if onNewBlock != nil { - onNewBlock(res.block.Hash) + onNewBlock(res.block.Hash, res.block.Height) } if res.block.Height > 0 && res.block.Height%1000 == 0 { glog.Info("connected block ", res.block.Height, " ", res.block.Hash) @@ -333,7 +334,7 @@ ConnectLoop: } hch <- hashHeight{hash, h} if h > 0 && h%1000 == 0 { - glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start)) + glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start), " ", w.db.GetAndResetConnectBlockStats()) start = time.Now() } if msTime.Before(time.Now()) { diff --git a/db/test_helper.go b/db/test_helper.go index 0fb16814..ce879684 100644 --- a/db/test_helper.go +++ b/db/test_helper.go @@ -2,10 +2,14 @@ package db -func ConnectBlocks(w *SyncWorker, onNewBlock func(hash string), initialSync bool) error { +import ( + "blockbook/bchain" +) + +func ConnectBlocks(w *SyncWorker, onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { return w.connectBlocks(onNewBlock, initialSync) } -func HandleFork(w *SyncWorker, localBestHeight uint32, localBestHash string, onNewBlock func(hash string), initialSync bool) error { +func HandleFork(w *SyncWorker, localBestHeight uint32, localBestHash string, onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { return w.handleFork(localBestHeight, localBestHash, onNewBlock, initialSync) } diff --git a/docs/config.md b/docs/config.md index af3f5cd0..61e76710 100644 --- a/docs/config.md +++ b/docs/config.md @@ -79,7 +79,7 @@ Good examples of coin configuration are * `system_user` – User used to run Blockbook service. See convention note in [build guide](/docs/build.md#on-naming-conventions-and-versioning). * `internal_binding_template` – Template for *-internal* parameter. See note on templates below. * `public_binding_template` – Template for *-public* parameter. See note on templates below. - * `explorer_url` – URL of blockchain explorer. + * `explorer_url` – URL of blockchain explorer. Leave empty for internal explorer. * `additional_params` – Additional params of exec command (see [Dogecoin definition](configs/coins/dogecoin.json)). * `block_chain` – Configuration of BlockChain type that ensures communication with back-end service. All options must be tweaked for each individual coin separely. diff --git a/server/internal.go b/server/internal.go index a4752b86..5e12dcf1 100644 --- a/server/internal.go +++ b/server/internal.go @@ -1,20 +1,17 @@ package server import ( + "blockbook/api" "blockbook/bchain" "blockbook/common" "blockbook/db" "context" "encoding/json" - "errors" "fmt" "net/http" - "strconv" - "time" "github.com/golang/glog" - "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -27,29 +24,21 @@ type InternalServer struct { chain bchain.BlockChain chainParser bchain.BlockChainParser is *common.InternalState -} - -type resAboutBlockbookInternal struct { - Coin string `json:"coin"` - Host string `json:"host"` - Version string `json:"version"` - GitCommit string `json:"gitcommit"` - BuildTime string `json:"buildtime"` - InSync bool `json:"inSync"` - BestHeight uint32 `json:"bestHeight"` - LastBlockTime time.Time `json:"lastBlockTime"` - InSyncMempool bool `json:"inSyncMempool"` - LastMempoolTime time.Time `json:"lastMempoolTime"` - MempoolSize int `json:"mempoolSize"` - DbColumns []common.InternalStateColumn `json:"dbColumns"` + api *api.Worker } // NewInternalServer creates new internal http interface to blockbook and returns its handle -func NewInternalServer(httpServerBinding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState) (*InternalServer, error) { - r := mux.NewRouter() +func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState) (*InternalServer, error) { + api, err := api.NewWorker(db, chain, txCache, is) + if err != nil { + return nil, err + } + + addr, path := splitBinding(binding) + serveMux := http.NewServeMux() https := &http.Server{ - Addr: httpServerBinding, - Handler: r, + Addr: addr, + Handler: serveMux, } s := &InternalServer{ https: https, @@ -59,15 +48,12 @@ func NewInternalServer(httpServerBinding string, certFiles string, db *db.RocksD chain: chain, chainParser: chain.GetChainParser(), is: is, + api: api, } - r.HandleFunc("/", s.index) - r.HandleFunc("/bestBlockHash", s.bestBlockHash) - r.HandleFunc("/blockHash/{height}", s.blockHash) - r.HandleFunc("/transactions/{address}/{lower}/{higher}", s.transactions) - r.HandleFunc("/confirmedTransactions/{address}/{lower}/{higher}", s.confirmedTransactions) - r.HandleFunc("/unconfirmedTransactions/{address}", s.unconfirmedTransactions) - r.HandleFunc("/metrics", promhttp.Handler().ServeHTTP) + serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) + serveMux.HandleFunc(path+"metrics", promhttp.Handler().ServeHTTP) + serveMux.HandleFunc(path, s.index) return s, nil } @@ -94,142 +80,11 @@ func (s *InternalServer) Shutdown(ctx context.Context) error { return s.https.Shutdown(ctx) } -func respondError(w http.ResponseWriter, err error, context string) { - w.WriteHeader(http.StatusBadRequest) - glog.Errorf("internal server: (context %s) error: %v", context, err) -} - -func respondHashData(w http.ResponseWriter, hash string) { - type hashData struct { - Hash string `json:"hash"` - } - json.NewEncoder(w).Encode(hashData{ - Hash: hash, - }) -} - func (s *InternalServer) index(w http.ResponseWriter, r *http.Request) { - vi := common.GetVersionInfo() - ss, bh, st := s.is.GetSyncState() - ms, mt, msz := s.is.GetMempoolSyncState() - a := resAboutBlockbookInternal{ - Coin: s.is.Coin, - Host: s.is.Host, - Version: vi.Version, - GitCommit: vi.GitCommit, - BuildTime: vi.BuildTime, - InSync: ss, - BestHeight: bh, - LastBlockTime: st, - InSyncMempool: ms, - LastMempoolTime: mt, - MempoolSize: msz, - DbColumns: s.is.GetAllDBColumnStats(), - } - buf, err := json.MarshalIndent(a, "", " ") + si, err := s.api.GetSystemInfo(true) + buf, err := json.MarshalIndent(si, "", " ") if err != nil { glog.Error(err) } w.Write(buf) } - -func (s *InternalServer) bestBlockHash(w http.ResponseWriter, r *http.Request) { - _, hash, err := s.db.GetBestBlock() - if err != nil { - respondError(w, err, "bestBlockHash") - return - } - respondHashData(w, hash) -} - -func (s *InternalServer) blockHash(w http.ResponseWriter, r *http.Request) { - heightString := mux.Vars(r)["height"] - var hash string - height, err := strconv.ParseUint(heightString, 10, 32) - if err == nil { - hash, err = s.db.GetBlockHash(uint32(height)) - } - if err != nil { - respondError(w, err, fmt.Sprintf("blockHash %s", heightString)) - } else { - respondHashData(w, hash) - } -} - -func (s *InternalServer) getAddress(r *http.Request) (address string, err error) { - address, ok := mux.Vars(r)["address"] - if !ok { - err = errors.New("Empty address") - } - return -} - -func (s *InternalServer) getAddressAndHeightRange(r *http.Request) (address string, lower, higher uint32, err error) { - address, err = s.getAddress(r) - if err != nil { - return - } - higher64, err := strconv.ParseUint(mux.Vars(r)["higher"], 10, 32) - if err != nil { - return - } - lower64, err := strconv.ParseUint(mux.Vars(r)["lower"], 10, 32) - if err != nil { - return - } - return address, uint32(lower64), uint32(higher64), err -} - -type transactionList struct { - Txid []string `json:"txid"` -} - -func (s *InternalServer) unconfirmedTransactions(w http.ResponseWriter, r *http.Request) { - address, err := s.getAddress(r) - if err != nil { - respondError(w, err, fmt.Sprint("unconfirmedTransactions for address", address)) - } - txs, err := s.chain.GetMempoolTransactions(address) - if err != nil { - respondError(w, err, fmt.Sprint("unconfirmedTransactions for address", address)) - } - txList := transactionList{Txid: txs} - json.NewEncoder(w).Encode(txList) -} - -func (s *InternalServer) confirmedTransactions(w http.ResponseWriter, r *http.Request) { - address, lower, higher, err := s.getAddressAndHeightRange(r) - if err != nil { - respondError(w, err, fmt.Sprint("confirmedTransactions for address", address)) - } - txList := transactionList{} - err = s.db.GetTransactions(address, lower, higher, func(txid string, vout uint32, isOutput bool) error { - txList.Txid = append(txList.Txid, txid) - return nil - }) - if err != nil { - respondError(w, err, fmt.Sprint("confirmedTransactions for address", address)) - } - json.NewEncoder(w).Encode(txList) -} - -func (s *InternalServer) transactions(w http.ResponseWriter, r *http.Request) { - address, lower, higher, err := s.getAddressAndHeightRange(r) - if err != nil { - respondError(w, err, fmt.Sprint("transactions for address", address)) - } - txList := transactionList{} - err = s.db.GetTransactions(address, lower, higher, func(txid string, vout uint32, isOutput bool) error { - txList.Txid = append(txList.Txid, txid) - return nil - }) - if err != nil { - respondError(w, err, fmt.Sprint("transactions for address", address)) - } - txs, err := s.chain.GetMempoolTransactions(address) - if err != nil { - respondError(w, err, fmt.Sprint("transactions for address", address)) - } - txList.Txid = append(txList.Txid, txs...) - json.NewEncoder(w).Encode(txList) -} diff --git a/server/public.go b/server/public.go index 7224f61a..b39aa796 100644 --- a/server/public.go +++ b/server/public.go @@ -19,29 +19,31 @@ import ( "github.com/golang/glog" ) -const blockbookAbout = "Blockbook - blockchain indexer for TREZOR wallet https://trezor.io/. Do not use for any other purpose." const txsOnPage = 25 +const blocksOnPage = 50 const txsInAPI = 1000 // PublicServer is a handle to public http server type PublicServer struct { - binding string - certFiles string - socketio *SocketIoServer - https *http.Server - db *db.RocksDB - txCache *db.TxCache - chain bchain.BlockChain - chainParser bchain.BlockChainParser - api *api.Worker - explorerURL string - metrics *common.Metrics - is *common.InternalState - templates []*template.Template - debug bool + binding string + certFiles string + socketio *SocketIoServer + https *http.Server + db *db.RocksDB + txCache *db.TxCache + chain bchain.BlockChain + chainParser bchain.BlockChainParser + api *api.Worker + explorerURL string + internalExplorer bool + metrics *common.Metrics + is *common.InternalState + templates []*template.Template + debug bool } // NewPublicServer creates new public server http interface to blockbook and returns its handle +// only basic functionality is mapped, to map all functions, call func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState, debugMode bool) (*PublicServer, error) { api, err := api.NewWorker(db, chain, txCache, is) @@ -62,43 +64,31 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch } s := &PublicServer{ - binding: binding, - certFiles: certFiles, - https: https, - api: api, - socketio: socketio, - db: db, - txCache: txCache, - chain: chain, - chainParser: chain.GetChainParser(), - explorerURL: explorerURL, - metrics: metrics, - is: is, - debug: debugMode, + binding: binding, + certFiles: certFiles, + https: https, + api: api, + socketio: socketio, + db: db, + txCache: txCache, + chain: chain, + chainParser: chain.GetChainParser(), + explorerURL: explorerURL, + internalExplorer: explorerURL == "", + metrics: metrics, + is: is, + debug: debugMode, } - - // favicon - serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) - // support for tests of socket.io interface - serveMux.Handle(path+"test.html", http.FileServer(http.Dir("./static/"))) - // redirect to wallet requests for tx and address, possibly to external site - serveMux.HandleFunc(path+"tx/", s.txRedirect) - serveMux.HandleFunc(path+"address/", s.addressRedirect) - // explorer - serveMux.HandleFunc(path+"explorer/tx/", s.htmlTemplateHandler(s.explorerTx)) - serveMux.HandleFunc(path+"explorer/address/", s.htmlTemplateHandler(s.explorerAddress)) - serveMux.Handle(path+"static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))) - // API calls - serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex)) - serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx)) - serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress)) - // handle socket.io - serveMux.Handle(path+"socket.io/", socketio.GetHandler()) - // default handler - serveMux.HandleFunc(path, s.index) - s.templates = parseTemplates() + // map only basic functions, the rest is enabled by method MapFullPublicInterface + serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) + serveMux.Handle(path+"static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))) + // default handler + serveMux.HandleFunc(path, s.htmlTemplateHandler(s.explorerIndex)) + // default API handler + serveMux.HandleFunc(path+"api/", s.jsonHandler(s.apiIndex)) + return s, nil } @@ -112,6 +102,34 @@ func (s *PublicServer) Run() error { return s.https.ListenAndServeTLS(fmt.Sprint(s.certFiles, ".crt"), fmt.Sprint(s.certFiles, ".key")) } +// ConnectFullPublicInterface enables complete public functionality +func (s *PublicServer) ConnectFullPublicInterface() { + serveMux := s.https.Handler.(*http.ServeMux) + _, path := splitBinding(s.binding) + // support for tests of socket.io interface + serveMux.Handle(path+"test.html", http.FileServer(http.Dir("./static/"))) + if s.internalExplorer { + // internal explorer handlers + serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx)) + serveMux.HandleFunc(path+"address/", s.htmlTemplateHandler(s.explorerAddress)) + serveMux.HandleFunc(path+"search/", s.htmlTemplateHandler(s.explorerSearch)) + serveMux.HandleFunc(path+"blocks", s.htmlTemplateHandler(s.explorerBlocks)) + serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock)) + serveMux.HandleFunc(path+"spending/", s.htmlTemplateHandler(s.explorerSpendingTx)) + } else { + // redirect to wallet requests for tx and address, possibly to external site + serveMux.HandleFunc(path+"tx/", s.txRedirect) + serveMux.HandleFunc(path+"address/", s.addressRedirect) + } + // API calls + serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex)) + serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx)) + serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress)) + serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock)) + // socket.io interface + serveMux.Handle(path+"socket.io/", s.socketio.GetHandler()) +} + // Close closes the server func (s *PublicServer) Close() error { glog.Infof("public server: closing") @@ -124,28 +142,24 @@ func (s *PublicServer) Shutdown(ctx context.Context) error { return s.https.Shutdown(ctx) } -// OnNewBlockHash notifies users subscribed to bitcoind/hashblock about new block -func (s *PublicServer) OnNewBlockHash(hash string) { +// OnNewBlock notifies users subscribed to bitcoind/hashblock about new block +func (s *PublicServer) OnNewBlock(hash string, height uint32) { s.socketio.OnNewBlockHash(hash) } // OnNewTxAddr notifies users subscribed to bitcoind/addresstxid about new block -func (s *PublicServer) OnNewTxAddr(txid string, addr string) { - s.socketio.OnNewTxAddr(txid, addr) +func (s *PublicServer) OnNewTxAddr(txid string, addr string, isOutput bool) { + s.socketio.OnNewTxAddr(txid, addr, isOutput) } func (s *PublicServer) txRedirect(w http.ResponseWriter, r *http.Request) { - if s.explorerURL != "" { - http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) - s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() - } + http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) + s.metrics.ExplorerViews.With(common.Labels{"action": "tx-redirect"}).Inc() } func (s *PublicServer) addressRedirect(w http.ResponseWriter, r *http.Request) { - if s.explorerURL != "" { - http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) - s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() - } + http.Redirect(w, r, joinURL(s.explorerURL, r.URL.Path), 302) + s.metrics.ExplorerViews.With(common.Labels{"action": "address-redirect"}).Inc() } func splitBinding(binding string) (addr string, path string) { @@ -187,6 +201,9 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, e } } w.Header().Set("Content-Type", "application/json; charset=utf-8") + if _, isError := data.(jsonError); isError { + w.WriteHeader(http.StatusInternalServerError) + } json.NewEncoder(w).Encode(data) }() data, err = handler(r) @@ -209,19 +226,19 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, e func (s *PublicServer) newTemplateData() *TemplateData { return &TemplateData{ - CoinName: s.is.Coin, - CoinShortcut: s.is.CoinShortcut, + CoinName: s.is.Coin, + CoinShortcut: s.is.CoinShortcut, + InternalExplorer: s.internalExplorer && !s.is.InitialSync, } } func (s *PublicServer) newTemplateDataWithError(text string) *TemplateData { - return &TemplateData{ - CoinName: s.is.Coin, - CoinShortcut: s.is.CoinShortcut, - Error: &api.ApiError{Text: text}, - } + td := s.newTemplateData() + td.Error = &api.ApiError{Text: text} + return td } -func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) { + +func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { var t tpl var data *TemplateData @@ -229,16 +246,23 @@ func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, * defer func() { if e := recover(); e != nil { glog.Error(getFunctionName(handler), " recovered from panic: ", e) - t = errorTpl + t = errorInternalTpl if s.debug { data = s.newTemplateDataWithError(fmt.Sprint("Internal server error: recovered from panic ", e)) } else { data = s.newTemplateDataWithError("Internal server error") } } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := s.templates[t].ExecuteTemplate(w, "base.html", data); err != nil { - glog.Error(err) + // noTpl means the handler completely handled the request + if t != noTpl { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + // return 500 Internal Server Error with errorInternalTpl + if t == errorInternalTpl { + w.WriteHeader(http.StatusInternalServerError) + } + if err := s.templates[t].ExecuteTemplate(w, "base.html", data); err != nil { + glog.Error(err) + } } }() if s.debug { @@ -246,12 +270,15 @@ func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, * // to reflect changes during development s.templates = parseTemplates() } - t, data, err = handler(r) - if err != nil || data == nil { - t = errorTpl + t, data, err = handler(w, r) + if err != nil || (data == nil && t != noTpl) { + t = errorInternalTpl if apiErr, ok := err.(*api.ApiError); ok { data = s.newTemplateData() data.Error = apiErr + if apiErr.Public { + t = errorTpl + } } else { if err != nil { glog.Error(getFunctionName(handler), " error: ", err) @@ -269,28 +296,38 @@ func (s *PublicServer) htmlTemplateHandler(handler func(r *http.Request) (tpl, * type tpl int const ( - errorTpl = tpl(iota) + noTpl = tpl(iota) + errorTpl + errorInternalTpl + indexTpl txTpl addressTpl + blocksTpl + blockTpl tplCount ) type TemplateData struct { - CoinName string - CoinShortcut string - Address *api.Address - AddrStr string - Tx *api.Tx - Error *api.ApiError - Page int - PrevPage int - NextPage int - PagingRange []int + CoinName string + CoinShortcut string + InternalExplorer bool + Address *api.Address + AddrStr string + Tx *api.Tx + Error *api.ApiError + Blocks *api.Blocks + Block *api.Block + Info *api.SystemInfo + Page int + PrevPage int + NextPage int + PagingRange []int } func parseTemplates() []*template.Template { templateFuncMap := template.FuncMap{ + "formatTime": formatTime, "formatUnixTime": formatUnixTime, "formatAmount": formatAmount, "setTxToTemplateData": setTxToTemplateData, @@ -298,13 +335,21 @@ func parseTemplates() []*template.Template { } t := make([]*template.Template, tplCount) t[errorTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) + t[errorInternalTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) + t[indexTpl] = template.Must(template.New("index").Funcs(templateFuncMap).ParseFiles("./static/templates/index.html", "./static/templates/base.html")) t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) + t[blocksTpl] = template.Must(template.New("blocks").Funcs(templateFuncMap).ParseFiles("./static/templates/blocks.html", "./static/templates/paging.html", "./static/templates/base.html")) + t[blockTpl] = template.Must(template.New("block").Funcs(templateFuncMap).ParseFiles("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) return t } func formatUnixTime(ut int64) string { - return time.Unix(ut, 0).Format(time.RFC1123) + return formatTime(time.Unix(ut, 0)) +} + +func formatTime(t time.Time) string { + return t.Format(time.RFC1123) } // for now return the string as it is @@ -319,13 +364,14 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { return td } -func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { +func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx + s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] bestheight, _, err := s.db.GetBestBlock() if err == nil { - tx, err = s.api.GetTransaction(txid, bestheight, true) + tx, err = s.api.GetTransaction(txid, bestheight, false) } if err != nil { return errorTpl, nil, err @@ -336,9 +382,31 @@ func (s *PublicServer) explorerTx(r *http.Request) (tpl, *TemplateData, error) { return txTpl, data, nil } -func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, error) { +func (s *PublicServer) explorerSpendingTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + s.metrics.ExplorerViews.With(common.Labels{"action": "spendingtx"}).Inc() + var err error + parts := strings.Split(r.URL.Path, "/") + if len(parts) > 2 { + tx := parts[len(parts)-2] + n, ec := strconv.Atoi(parts[len(parts)-1]) + if ec == nil { + spendingTx, err := s.api.GetSpendingTxid(tx, n) + if err == nil && spendingTx != "" { + http.Redirect(w, r, joinURL("/tx/", spendingTx), 302) + return noTpl, nil, nil + } + } + } + if err == nil { + err = api.NewApiError("Transaction not found", true) + } + return errorTpl, nil, err +} + +func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var address *api.Address var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { @@ -357,6 +425,90 @@ func (s *PublicServer) explorerAddress(r *http.Request) (tpl, *TemplateData, err return addressTpl, data, nil } +func (s *PublicServer) explorerBlocks(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var blocks *api.Blocks + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "blocks"}).Inc() + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + blocks, err = s.api.GetBlocks(page, blocksOnPage) + if err != nil { + return errorTpl, nil, err + } + data := s.newTemplateData() + data.Blocks = blocks + data.Page = blocks.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(blocks.Page, blocks.TotalPages) + return blocksTpl, data, nil +} + +func (s *PublicServer) explorerBlock(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var block *api.Block + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "block"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + block, err = s.api.GetBlock(r.URL.Path[i+1:], page, txsOnPage) + if err != nil { + return errorTpl, nil, err + } + } + data := s.newTemplateData() + data.Block = block + data.Page = block.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(block.Page, block.TotalPages) + return blockTpl, data, nil +} + +func (s *PublicServer) explorerIndex(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var si *api.SystemInfo + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "index"}).Inc() + si, err = s.api.GetSystemInfo(false) + if err != nil { + return errorTpl, nil, err + } + data := s.newTemplateData() + data.Info = si + return indexTpl, data, nil +} + +func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + q := strings.TrimSpace(r.URL.Query().Get("q")) + var tx *api.Tx + var address *api.Address + var block *api.Block + var bestheight uint32 + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "search"}).Inc() + if len(q) > 0 { + block, err = s.api.GetBlock(q, 0, 1) + if err == nil { + http.Redirect(w, r, joinURL("/block/", block.Hash), 302) + return noTpl, nil, nil + } + bestheight, _, err = s.db.GetBestBlock() + if err == nil { + tx, err = s.api.GetTransaction(q, bestheight, false) + if err == nil { + http.Redirect(w, r, joinURL("/tx/", tx.Txid), 302) + return noTpl, nil, nil + } + } + address, err = s.api.GetAddress(q, 0, 1, true) + if err == nil { + http.Redirect(w, r, joinURL("/address/", address.AddrStr), 302) + return noTpl, nil, nil + } + } + return errorTpl, nil, api.NewApiError(fmt.Sprintf("No matching records found for '%v'", q), true) +} + func getPagingRange(page int, total int) ([]int, int, int) { if total < 2 { return nil, 0, 0 @@ -411,50 +563,14 @@ func getPagingRange(page int, total int) ([]int, int, int) { return r, pp, np } -type resAboutBlockbookPublic struct { - Coin string `json:"coin"` - Host string `json:"host"` - Version string `json:"version"` - GitCommit string `json:"gitcommit"` - BuildTime string `json:"buildtime"` - InSync bool `json:"inSync"` - BestHeight uint32 `json:"bestHeight"` - LastBlockTime time.Time `json:"lastBlockTime"` - InSyncMempool bool `json:"inSyncMempool"` - LastMempoolTime time.Time `json:"lastMempoolTime"` - About string `json:"about"` -} - -// TODO - this is temporary, return html status page -func (s *PublicServer) index(w http.ResponseWriter, r *http.Request) { - vi := common.GetVersionInfo() - ss, bh, st := s.is.GetSyncState() - ms, mt, _ := s.is.GetMempoolSyncState() - a := resAboutBlockbookPublic{ - Coin: s.is.Coin, - Host: s.is.Host, - Version: vi.Version, - GitCommit: vi.GitCommit, - BuildTime: vi.BuildTime, - InSync: ss, - BestHeight: bh, - LastBlockTime: st, - InSyncMempool: ms, - LastMempoolTime: mt, - About: blockbookAbout, - } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - buf, err := json.MarshalIndent(a, "", " ") - if err != nil { - glog.Error(err) - } - w.Write(buf) +func (s *PublicServer) apiIndex(r *http.Request) (interface{}, error) { + s.metrics.ExplorerViews.With(common.Labels{"action": "api-index"}).Inc() + return s.api.GetSystemInfo(false) } func (s *PublicServer) apiBlockIndex(r *http.Request) (interface{}, error) { type resBlockIndex struct { BlockHash string `json:"blockHash"` - About string `json:"about"` } var err error var hash string @@ -475,13 +591,13 @@ func (s *PublicServer) apiBlockIndex(r *http.Request) (interface{}, error) { } return resBlockIndex{ BlockHash: hash, - About: blockbookAbout, }, nil } func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) { var tx *api.Tx var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-tx"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] bestheight, _, err := s.db.GetBestBlock() @@ -495,6 +611,7 @@ func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) { func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { var address *api.Address var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-address"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { page, ec := strconv.Atoi(r.URL.Query().Get("page")) if ec != nil { @@ -504,3 +621,17 @@ func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { } return address, err } + +func (s *PublicServer) apiBlock(r *http.Request) (interface{}, error) { + var block *api.Block + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "api-block"}).Inc() + if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + block, err = s.api.GetBlock(r.URL.Path[i+1:], page, txsInAPI) + } + return block, err +} diff --git a/server/socketio.go b/server/socketio.go index f9924452..2e931d1f 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -551,7 +551,7 @@ func (s *SocketIoServer) getInfo() (res resultGetInfo, err error) { res.Result.Network = s.chain.GetNetworkName() res.Result.Subversion = s.chain.GetSubversion() res.Result.CoinName = s.chain.GetCoinName() - res.Result.About = blockbookAbout + res.Result.About = api.BlockbookAbout return } @@ -748,8 +748,12 @@ func (s *SocketIoServer) OnNewBlockHash(hash string) { } // OnNewTxAddr notifies users subscribed to bitcoind/addresstxid about new block -func (s *SocketIoServer) OnNewTxAddr(txid string, addr string) { - c := s.server.BroadcastTo("bitcoind/addresstxid-"+addr, "bitcoind/addresstxid", map[string]string{"address": addr, "txid": txid}) +func (s *SocketIoServer) OnNewTxAddr(txid string, addr string, isOutput bool) { + data := map[string]interface{}{"address": addr, "txid": txid} + if !isOutput { + data["input"] = true + } + c := s.server.BroadcastTo("bitcoind/addresstxid-"+addr, "bitcoind/addresstxid", data) if c > 0 { glog.Info("broadcasting new txid ", txid, " for addr ", addr, " to ", c, " channels") } diff --git a/static/css/main.css b/static/css/main.css index f2ad1378..b18df624 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -31,6 +31,22 @@ h3 { height: 16px; } +.navbar-form { + padding-bottom: 1px; +} + +.navbar-form .form-control { + background-color: gray; + color: #fff; + border-radius: 3px; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border: 0; + -webkit-box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); + -moz-box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); + box-shadow: 1px 1px 0 0 rgba(255, 255, 255, .41), inset 1px 1px 3px 0 rgba(0, 0, 0, .10); +} + @media (min-width: 768px) { .container { max-width: 750px; @@ -47,6 +63,9 @@ h3 { .octicon { height: 24px; } + .navbar-form .form-control { + width: 230px; + } } @media (min-width: 1200px) { @@ -59,6 +78,9 @@ h3 { .octicon { height: 32px; } + .navbar-form .form-control { + width: 360px; + } } #header { @@ -83,7 +105,6 @@ h3 { color: rgba(255, 255, 255); font-weight: bold; font-size: 19px; - padding-left: 35px; } .trezor-logo-svg-white svg { @@ -114,7 +135,7 @@ h3 { .line-top { border-top: 1px solid #EAEAEA; - padding: 15px 0 0; + padding: 10px 0 0; } .line-mid { @@ -136,6 +157,11 @@ h3 { white-space: nowrap; vertical-align: baseline; border-radius: .25em; + margin-top: 5px; +} + +.txvalues:not(:last-child) { + margin-right: 5px; } .txvalues-default { @@ -157,14 +183,17 @@ h3 { } .ellipsis { - display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .data-div { - margin: 20px 0; + margin: 20px 0 30px 0; +} + +.data-div .col-md-10 { + padding-left: 0; } .data-table { @@ -194,6 +223,22 @@ h3 { padding-left: .25rem; } +.navbar-text .nav-link { + padding: 0; +} + +::-webkit-input-placeholder { + color: #CCC!important; + font-style: italic; + font-size: 14px; +} + +::-moz-placeholder { + color: #CCC!important; + font-style: italic; + font-size: 14px; +} + .h-container { display: -webkit-box; display: -ms-flexbox; diff --git a/static/js/qrcode.min.js b/static/js/qrcode.min.js new file mode 100755 index 00000000..993e88f3 --- /dev/null +++ b/static/js/qrcode.min.js @@ -0,0 +1 @@ +var QRCode;!function(){function a(a){this.mode=c.MODE_8BIT_BYTE,this.data=a,this.parsedData=[];for(var b=[],d=0,e=this.data.length;e>d;d++){var f=this.data.charCodeAt(d);f>65536?(b[0]=240|(1835008&f)>>>18,b[1]=128|(258048&f)>>>12,b[2]=128|(4032&f)>>>6,b[3]=128|63&f):f>2048?(b[0]=224|(61440&f)>>>12,b[1]=128|(4032&f)>>>6,b[2]=128|63&f):f>128?(b[0]=192|(1984&f)>>>6,b[1]=128|63&f):b[0]=f,this.parsedData=this.parsedData.concat(b)}this.parsedData.length!=this.data.length&&(this.parsedData.unshift(191),this.parsedData.unshift(187),this.parsedData.unshift(239))}function b(a,b){this.typeNumber=a,this.errorCorrectLevel=b,this.modules=null,this.moduleCount=0,this.dataCache=null,this.dataList=[]}function i(a,b){if(void 0==a.length)throw new Error(a.length+"/"+b);for(var c=0;c=f;f++){var h=0;switch(b){case d.L:h=l[f][0];break;case d.M:h=l[f][1];break;case d.Q:h=l[f][2];break;case d.H:h=l[f][3]}if(h>=e)break;c++}if(c>l.length)throw new Error("Too long data");return c}function s(a){var b=encodeURI(a).toString().replace(/\%[0-9a-fA-F]{2}/g,"a");return b.length+(b.length!=a?3:0)}a.prototype={getLength:function(){return this.parsedData.length},write:function(a){for(var b=0,c=this.parsedData.length;c>b;b++)a.put(this.parsedData[b],8)}},b.prototype={addData:function(b){var c=new a(b);this.dataList.push(c),this.dataCache=null},isDark:function(a,b){if(0>a||this.moduleCount<=a||0>b||this.moduleCount<=b)throw new Error(a+","+b);return this.modules[a][b]},getModuleCount:function(){return this.moduleCount},make:function(){this.makeImpl(!1,this.getBestMaskPattern())},makeImpl:function(a,c){this.moduleCount=4*this.typeNumber+17,this.modules=new Array(this.moduleCount);for(var d=0;d=7&&this.setupTypeNumber(a),null==this.dataCache&&(this.dataCache=b.createData(this.typeNumber,this.errorCorrectLevel,this.dataList)),this.mapData(this.dataCache,c)},setupPositionProbePattern:function(a,b){for(var c=-1;7>=c;c++)if(!(-1>=a+c||this.moduleCount<=a+c))for(var d=-1;7>=d;d++)-1>=b+d||this.moduleCount<=b+d||(this.modules[a+c][b+d]=c>=0&&6>=c&&(0==d||6==d)||d>=0&&6>=d&&(0==c||6==c)||c>=2&&4>=c&&d>=2&&4>=d?!0:!1)},getBestMaskPattern:function(){for(var a=0,b=0,c=0;8>c;c++){this.makeImpl(!0,c);var d=f.getLostPoint(this);(0==c||a>d)&&(a=d,b=c)}return b},createMovieClip:function(a,b,c){var d=a.createEmptyMovieClip(b,c),e=1;this.make();for(var f=0;f=g;g++)for(var h=-2;2>=h;h++)this.modules[d+g][e+h]=-2==g||2==g||-2==h||2==h||0==g&&0==h?!0:!1}},setupTypeNumber:function(a){for(var b=f.getBCHTypeNumber(this.typeNumber),c=0;18>c;c++){var d=!a&&1==(1&b>>c);this.modules[Math.floor(c/3)][c%3+this.moduleCount-8-3]=d}for(var c=0;18>c;c++){var d=!a&&1==(1&b>>c);this.modules[c%3+this.moduleCount-8-3][Math.floor(c/3)]=d}},setupTypeInfo:function(a,b){for(var c=this.errorCorrectLevel<<3|b,d=f.getBCHTypeInfo(c),e=0;15>e;e++){var g=!a&&1==(1&d>>e);6>e?this.modules[e][8]=g:8>e?this.modules[e+1][8]=g:this.modules[this.moduleCount-15+e][8]=g}for(var e=0;15>e;e++){var g=!a&&1==(1&d>>e);8>e?this.modules[8][this.moduleCount-e-1]=g:9>e?this.modules[8][15-e-1+1]=g:this.modules[8][15-e-1]=g}this.modules[this.moduleCount-8][8]=!a},mapData:function(a,b){for(var c=-1,d=this.moduleCount-1,e=7,g=0,h=this.moduleCount-1;h>0;h-=2)for(6==h&&h--;;){for(var i=0;2>i;i++)if(null==this.modules[d][h-i]){var j=!1;g>>e));var k=f.getMask(b,d,h-i);k&&(j=!j),this.modules[d][h-i]=j,e--,-1==e&&(g++,e=7)}if(d+=c,0>d||this.moduleCount<=d){d-=c,c=-c;break}}}},b.PAD0=236,b.PAD1=17,b.createData=function(a,c,d){for(var e=j.getRSBlocks(a,c),g=new k,h=0;h8*l)throw new Error("code length overflow. ("+g.getLengthInBits()+">"+8*l+")");for(g.getLengthInBits()+4<=8*l&&g.put(0,4);0!=g.getLengthInBits()%8;)g.putBit(!1);for(;;){if(g.getLengthInBits()>=8*l)break;if(g.put(b.PAD0,8),g.getLengthInBits()>=8*l)break;g.put(b.PAD1,8)}return b.createBytes(g,e)},b.createBytes=function(a,b){for(var c=0,d=0,e=0,g=new Array(b.length),h=new Array(b.length),j=0;j=0?p.get(q):0}}for(var r=0,m=0;mm;m++)for(var j=0;jm;m++)for(var j=0;j=0;)b^=f.G15<=0;)b^=f.G18<>>=1;return b},getPatternPosition:function(a){return f.PATTERN_POSITION_TABLE[a-1]},getMask:function(a,b,c){switch(a){case e.PATTERN000:return 0==(b+c)%2;case e.PATTERN001:return 0==b%2;case e.PATTERN010:return 0==c%3;case e.PATTERN011:return 0==(b+c)%3;case e.PATTERN100:return 0==(Math.floor(b/2)+Math.floor(c/3))%2;case e.PATTERN101:return 0==b*c%2+b*c%3;case e.PATTERN110:return 0==(b*c%2+b*c%3)%2;case e.PATTERN111:return 0==(b*c%3+(b+c)%2)%2;default:throw new Error("bad maskPattern:"+a)}},getErrorCorrectPolynomial:function(a){for(var b=new i([1],0),c=0;a>c;c++)b=b.multiply(new i([1,g.gexp(c)],0));return b},getLengthInBits:function(a,b){if(b>=1&&10>b)switch(a){case c.MODE_NUMBER:return 10;case c.MODE_ALPHA_NUM:return 9;case c.MODE_8BIT_BYTE:return 8;case c.MODE_KANJI:return 8;default:throw new Error("mode:"+a)}else if(27>b)switch(a){case c.MODE_NUMBER:return 12;case c.MODE_ALPHA_NUM:return 11;case c.MODE_8BIT_BYTE:return 16;case c.MODE_KANJI:return 10;default:throw new Error("mode:"+a)}else{if(!(41>b))throw new Error("type:"+b);switch(a){case c.MODE_NUMBER:return 14;case c.MODE_ALPHA_NUM:return 13;case c.MODE_8BIT_BYTE:return 16;case c.MODE_KANJI:return 12;default:throw new Error("mode:"+a)}}},getLostPoint:function(a){for(var b=a.getModuleCount(),c=0,d=0;b>d;d++)for(var e=0;b>e;e++){for(var f=0,g=a.isDark(d,e),h=-1;1>=h;h++)if(!(0>d+h||d+h>=b))for(var i=-1;1>=i;i++)0>e+i||e+i>=b||(0!=h||0!=i)&&g==a.isDark(d+h,e+i)&&f++;f>5&&(c+=3+f-5)}for(var d=0;b-1>d;d++)for(var e=0;b-1>e;e++){var j=0;a.isDark(d,e)&&j++,a.isDark(d+1,e)&&j++,a.isDark(d,e+1)&&j++,a.isDark(d+1,e+1)&&j++,(0==j||4==j)&&(c+=3)}for(var d=0;b>d;d++)for(var e=0;b-6>e;e++)a.isDark(d,e)&&!a.isDark(d,e+1)&&a.isDark(d,e+2)&&a.isDark(d,e+3)&&a.isDark(d,e+4)&&!a.isDark(d,e+5)&&a.isDark(d,e+6)&&(c+=40);for(var e=0;b>e;e++)for(var d=0;b-6>d;d++)a.isDark(d,e)&&!a.isDark(d+1,e)&&a.isDark(d+2,e)&&a.isDark(d+3,e)&&a.isDark(d+4,e)&&!a.isDark(d+5,e)&&a.isDark(d+6,e)&&(c+=40);for(var k=0,e=0;b>e;e++)for(var d=0;b>d;d++)a.isDark(d,e)&&k++;var l=Math.abs(100*k/b/b-50)/5;return c+=10*l}},g={glog:function(a){if(1>a)throw new Error("glog("+a+")");return g.LOG_TABLE[a]},gexp:function(a){for(;0>a;)a+=255;for(;a>=256;)a-=255;return g.EXP_TABLE[a]},EXP_TABLE:new Array(256),LOG_TABLE:new Array(256)},h=0;8>h;h++)g.EXP_TABLE[h]=1<h;h++)g.EXP_TABLE[h]=g.EXP_TABLE[h-4]^g.EXP_TABLE[h-5]^g.EXP_TABLE[h-6]^g.EXP_TABLE[h-8];for(var h=0;255>h;h++)g.LOG_TABLE[g.EXP_TABLE[h]]=h;i.prototype={get:function(a){return this.num[a]},getLength:function(){return this.num.length},multiply:function(a){for(var b=new Array(this.getLength()+a.getLength()-1),c=0;cf;f++)for(var g=c[3*f+0],h=c[3*f+1],i=c[3*f+2],k=0;g>k;k++)e.push(new j(h,i));return e},j.getRsBlockTable=function(a,b){switch(b){case d.L:return j.RS_BLOCK_TABLE[4*(a-1)+0];case d.M:return j.RS_BLOCK_TABLE[4*(a-1)+1];case d.Q:return j.RS_BLOCK_TABLE[4*(a-1)+2];case d.H:return j.RS_BLOCK_TABLE[4*(a-1)+3];default:return void 0}},k.prototype={get:function(a){var b=Math.floor(a/8);return 1==(1&this.buffer[b]>>>7-a%8)},put:function(a,b){for(var c=0;b>c;c++)this.putBit(1==(1&a>>>b-c-1))},getLengthInBits:function(){return this.length},putBit:function(a){var b=Math.floor(this.length/8);this.buffer.length<=b&&this.buffer.push(0),a&&(this.buffer[b]|=128>>>this.length%8),this.length++}};var l=[[17,14,11,7],[32,26,20,14],[53,42,32,24],[78,62,46,34],[106,84,60,44],[134,106,74,58],[154,122,86,64],[192,152,108,84],[230,180,130,98],[271,213,151,119],[321,251,177,137],[367,287,203,155],[425,331,241,177],[458,362,258,194],[520,412,292,220],[586,450,322,250],[644,504,364,280],[718,560,394,310],[792,624,442,338],[858,666,482,382],[929,711,509,403],[1003,779,565,439],[1091,857,611,461],[1171,911,661,511],[1273,997,715,535],[1367,1059,751,593],[1465,1125,805,625],[1528,1190,868,658],[1628,1264,908,698],[1732,1370,982,742],[1840,1452,1030,790],[1952,1538,1112,842],[2068,1628,1168,898],[2188,1722,1228,958],[2303,1809,1283,983],[2431,1911,1351,1051],[2563,1989,1423,1093],[2699,2099,1499,1139],[2809,2213,1579,1219],[2953,2331,1663,1273]],o=function(){var a=function(a,b){this._el=a,this._htOption=b};return a.prototype.draw=function(a){function g(a,b){var c=document.createElementNS("http://www.w3.org/2000/svg",a);for(var d in b)b.hasOwnProperty(d)&&c.setAttribute(d,b[d]);return c}var b=this._htOption,c=this._el,d=a.getModuleCount();Math.floor(b.width/d),Math.floor(b.height/d),this.clear();var h=g("svg",{viewBox:"0 0 "+String(d)+" "+String(d),width:"100%",height:"100%",fill:b.colorLight});h.setAttributeNS("http://www.w3.org/2000/xmlns/","xmlns:xlink","http://www.w3.org/1999/xlink"),c.appendChild(h),h.appendChild(g("rect",{fill:b.colorDark,width:"1",height:"1",id:"template"}));for(var i=0;d>i;i++)for(var j=0;d>j;j++)if(a.isDark(i,j)){var k=g("use",{x:String(i),y:String(j)});k.setAttributeNS("http://www.w3.org/1999/xlink","href","#template"),h.appendChild(k)}},a.prototype.clear=function(){for(;this._el.hasChildNodes();)this._el.removeChild(this._el.lastChild)},a}(),p="svg"===document.documentElement.tagName.toLowerCase(),q=p?o:m()?function(){function a(){this._elImage.src=this._elCanvas.toDataURL("image/png"),this._elImage.style.display="block",this._elCanvas.style.display="none"}function d(a,b){var c=this;if(c._fFail=b,c._fSuccess=a,null===c._bSupportDataURI){var d=document.createElement("img"),e=function(){c._bSupportDataURI=!1,c._fFail&&_fFail.call(c)},f=function(){c._bSupportDataURI=!0,c._fSuccess&&c._fSuccess.call(c)};return d.onabort=e,d.onerror=e,d.onload=f,d.src="data:image/gif;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==",void 0}c._bSupportDataURI===!0&&c._fSuccess?c._fSuccess.call(c):c._bSupportDataURI===!1&&c._fFail&&c._fFail.call(c)}if(this._android&&this._android<=2.1){var b=1/window.devicePixelRatio,c=CanvasRenderingContext2D.prototype.drawImage;CanvasRenderingContext2D.prototype.drawImage=function(a,d,e,f,g,h,i,j){if("nodeName"in a&&/img/i.test(a.nodeName))for(var l=arguments.length-1;l>=1;l--)arguments[l]=arguments[l]*b;else"undefined"==typeof j&&(arguments[1]*=b,arguments[2]*=b,arguments[3]*=b,arguments[4]*=b);c.apply(this,arguments)}}var e=function(a,b){this._bIsPainted=!1,this._android=n(),this._htOption=b,this._elCanvas=document.createElement("canvas"),this._elCanvas.width=b.width,this._elCanvas.height=b.height,a.appendChild(this._elCanvas),this._el=a,this._oContext=this._elCanvas.getContext("2d"),this._bIsPainted=!1,this._elImage=document.createElement("img"),this._elImage.style.display="none",this._el.appendChild(this._elImage),this._bSupportDataURI=null};return e.prototype.draw=function(a){var b=this._elImage,c=this._oContext,d=this._htOption,e=a.getModuleCount(),f=d.width/e,g=d.height/e,h=Math.round(f),i=Math.round(g);b.style.display="none",this.clear();for(var j=0;e>j;j++)for(var k=0;e>k;k++){var l=a.isDark(j,k),m=k*f,n=j*g;c.strokeStyle=l?d.colorDark:d.colorLight,c.lineWidth=1,c.fillStyle=l?d.colorDark:d.colorLight,c.fillRect(m,n,f,g),c.strokeRect(Math.floor(m)+.5,Math.floor(n)+.5,h,i),c.strokeRect(Math.ceil(m)-.5,Math.ceil(n)-.5,h,i)}this._bIsPainted=!0},e.prototype.makeImage=function(){this._bIsPainted&&d.call(this,a)},e.prototype.isPainted=function(){return this._bIsPainted},e.prototype.clear=function(){this._oContext.clearRect(0,0,this._elCanvas.width,this._elCanvas.height),this._bIsPainted=!1},e.prototype.round=function(a){return a?Math.floor(1e3*a)/1e3:a},e}():function(){var a=function(a,b){this._el=a,this._htOption=b};return a.prototype.draw=function(a){for(var b=this._htOption,c=this._el,d=a.getModuleCount(),e=Math.floor(b.width/d),f=Math.floor(b.height/d),g=[''],h=0;d>h;h++){g.push("");for(var i=0;d>i;i++)g.push('');g.push("")}g.push("
"),c.innerHTML=g.join("");var j=c.childNodes[0],k=(b.width-j.offsetWidth)/2,l=(b.height-j.offsetHeight)/2;k>0&&l>0&&(j.style.margin=l+"px "+k+"px")},a.prototype.clear=function(){this._el.innerHTML=""},a}();QRCode=function(a,b){if(this._htOption={width:256,height:256,typeNumber:4,colorDark:"#000000",colorLight:"#ffffff",correctLevel:d.H},"string"==typeof b&&(b={text:b}),b)for(var c in b)this._htOption[c]=b[c];"string"==typeof a&&(a=document.getElementById(a)),this._android=n(),this._el=a,this._oQRCode=null,this._oDrawing=new q(this._el,this._htOption),this._htOption.text&&this.makeCode(this._htOption.text)},QRCode.prototype.makeCode=function(a){this._oQRCode=new b(r(a,this._htOption.correctLevel),this._htOption.correctLevel),this._oQRCode.addData(a),this._oQRCode.make(),this._el.title=a,this._oDrawing.draw(this._oQRCode),this.makeImage()},QRCode.prototype.makeImage=function(){"function"==typeof this._oDrawing.makeImage&&(!this._android||this._android>=3)&&this._oDrawing.makeImage()},QRCode.prototype.clear=function(){this._oDrawing.clear()},QRCode.CorrectLevel=d}(); \ No newline at end of file diff --git a/static/templates/address.html b/static/templates/address.html index b9100cb7..7a56e729 100644 --- a/static/templates/address.html +++ b/static/templates/address.html @@ -2,31 +2,40 @@

Address {{formatAmount $addr.Balance}} {{$cs}}

-
- {{$addr.AddrStr}} +
+ {{$addr.AddrStr}}

Confirmed

-
- - - - - - - - - - - - - - - - - - - -
Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}
Final Balance{{formatAmount $addr.Balance}} {{$cs}}
No. Transactions{{$addr.TxApperances}}
+
+
+ + + + + + + + + + + + + + + + + + + +
Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}
Final Balance{{formatAmount $addr.Balance}} {{$cs}}
No. Transactions{{$addr.TxApperances}}
+
+
+
+ + +
{{- if $addr.UnconfirmedTxApperances -}}

Unconfirmed

diff --git a/static/templates/base.html b/static/templates/base.html index 5ab2fb3e..05bebd41 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -14,9 +14,8 @@
- {{template "specific" .}} + {{- template "specific" . -}}