From ce3c7c5e6678107e7d9c09264754a25e7286e592 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 25 Mar 2019 16:43:57 +0100 Subject: [PATCH 01/39] Extract mempool interface from blockchain --- api/worker.go | 6 +- api/xpub.go | 2 +- bchain/basechain.go | 5 ++ bchain/coins/bch/bcashrpc.go | 3 +- bchain/coins/bellcoin/bellcoinrpc.go | 3 +- bchain/coins/blockchain.go | 85 ++++++++++++++-------- bchain/coins/btc/bitcoinrpc.go | 75 ++++++++----------- bchain/coins/btg/bgoldrpc.go | 3 +- bchain/coins/dash/dashrpc.go | 3 +- bchain/coins/digibyte/digibyterpc.go | 3 +- bchain/coins/dogecoin/dogecoinrpc.go | 3 +- bchain/coins/eth/ethrpc.go | 48 +++++------- bchain/coins/flo/florpc.go | 4 +- bchain/coins/fujicoin/fujicoinrpc.go | 3 +- bchain/coins/gamecredits/gamecreditsrpc.go | 3 +- bchain/coins/grs/grsrpc.go | 3 +- bchain/coins/koto/kotorpc.go | 3 +- bchain/coins/liquid/liquidrpc.go | 3 +- bchain/coins/litecoin/litecoinrpc.go | 3 +- bchain/coins/monacoin/monacoinrpc.go | 3 +- bchain/coins/myriad/myriadrpc.go | 3 +- bchain/coins/namecoin/namecoinrpc.go | 3 +- bchain/coins/pivx/pivxrpc.go | 3 +- bchain/coins/qtum/qtumrpc.go | 4 +- bchain/coins/vertcoin/vertcoinrpc.go | 3 +- bchain/coins/xzc/zcoinrpc.go | 3 +- bchain/coins/zec/zcashrpc.go | 3 +- bchain/mempool_bitcoin_type.go | 2 +- bchain/mempool_ethereum_type.go | 2 +- bchain/types.go | 19 +++-- blockbook.go | 80 +++++++++++--------- server/internal.go | 6 +- server/public.go | 10 ++- server/public_test.go | 7 +- server/socketio.go | 8 +- server/websocket.go | 6 +- tests/dbtestdata/fakechain.go | 34 ++++----- tests/integration.go | 46 +++++++----- tests/rpc/rpc.go | 15 ++-- tests/sync/sync.go | 2 +- 40 files changed, 298 insertions(+), 225 deletions(-) diff --git a/api/worker.go b/api/worker.go index c8632758..806bcf2c 100644 --- a/api/worker.go +++ b/api/worker.go @@ -23,17 +23,19 @@ type Worker struct { chain bchain.BlockChain chainParser bchain.BlockChainParser chainType bchain.ChainType + mempool bchain.Mempool is *common.InternalState } // NewWorker creates new api worker -func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState) (*Worker, error) { +func NewWorker(db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, txCache *db.TxCache, is *common.InternalState) (*Worker, error) { w := &Worker{ db: db, txCache: txCache, chain: chain, chainParser: chain.GetChainParser(), chainType: chain.GetChainParser().GetChainType(), + mempool: mempool, is: is, } return w, nil @@ -348,7 +350,7 @@ func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool } if mempool { uniqueTxs := make(map[string]struct{}) - o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) + o, err := w.mempool.GetAddrDescTransactions(addrDesc) if err != nil { return nil, err } diff --git a/api/xpub.go b/api/xpub.go index 79bb2fe5..7508f24a 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -92,7 +92,7 @@ func (w *Worker) xpubGetAddressTxids(addrDesc bchain.AddressDescriptor, mempool } if mempool { uniqueTxs := make(map[string]int) - o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) + o, err := w.mempool.GetAddrDescTransactions(addrDesc) if err != nil { return nil, false, err } diff --git a/bchain/basechain.go b/bchain/basechain.go index 1de31a48..f1a58e35 100644 --- a/bchain/basechain.go +++ b/bchain/basechain.go @@ -29,6 +29,11 @@ func (b *BaseChain) GetNetworkName() string { return b.Network } +// GetMempoolEntry is not supported by default +func (b *BaseChain) GetMempoolEntry(txid string) (*MempoolEntry, error) { + return nil, errors.New("GetMempoolEntry: not supported") +} + // EthereumTypeGetBalance is not supported func (b *BaseChain) EthereumTypeGetBalance(addrDesc AddressDescriptor) (*big.Int, error) { return nil, errors.New("Not supported") diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 9a3c7d4b..36a955c1 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -33,10 +33,11 @@ func NewBCashRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp // Initialize initializes BCashRPC instance. func (b *BCashRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/bellcoin/bellcoinrpc.go b/bchain/coins/bellcoin/bellcoinrpc.go index bd359fd2..2317e063 100644 --- a/bchain/coins/bellcoin/bellcoinrpc.go +++ b/bchain/coins/bellcoin/bellcoinrpc.go @@ -31,10 +31,11 @@ func NewBellcoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes BellcoinRPC instance. func (b *BellcoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 4c251f59..34871e75 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -99,30 +99,34 @@ func GetCoinNameFromConfig(configfile string) (string, string, string, error) { return cn.CoinName, cn.CoinShortcut, cn.CoinLabel, nil } -// NewBlockChain creates bchain.BlockChain of type defined by parameter coin -func NewBlockChain(coin string, configfile string, pushHandler func(bchain.NotificationType), metrics *common.Metrics) (bchain.BlockChain, error) { +// NewBlockChain creates bchain.BlockChain and bchain.Mempool for the coin passed by the parameter coin +func NewBlockChain(coin string, configfile string, pushHandler func(bchain.NotificationType), metrics *common.Metrics) (bchain.BlockChain, bchain.Mempool, error) { data, err := ioutil.ReadFile(configfile) if err != nil { - return nil, errors.Annotatef(err, "Error reading file %v", configfile) + return nil, nil, errors.Annotatef(err, "Error reading file %v", configfile) } var config json.RawMessage err = json.Unmarshal(data, &config) if err != nil { - return nil, errors.Annotatef(err, "Error parsing file %v", configfile) + return nil, nil, errors.Annotatef(err, "Error parsing file %v", configfile) } bcf, ok := BlockChainFactories[coin] if !ok { - return nil, errors.New(fmt.Sprint("Unsupported coin '", coin, "'. Must be one of ", reflect.ValueOf(BlockChainFactories).MapKeys())) + return nil, nil, errors.New(fmt.Sprint("Unsupported coin '", coin, "'. Must be one of ", reflect.ValueOf(BlockChainFactories).MapKeys())) } bc, err := bcf(config, pushHandler) if err != nil { - return nil, err + return nil, nil, err } err = bc.Initialize() if err != nil { - return nil, err + return nil, nil, err } - return &blockChainWithMetrics{b: bc, m: metrics}, nil + mempool, err := bc.CreateMempool() + if err != nil { + return nil, nil, err + } + return &blockChainWithMetrics{b: bc, m: metrics}, &mempoolWithMetrics{mempool: mempool, m: metrics}, nil } type blockChainWithMetrics struct { @@ -142,6 +146,14 @@ func (c *blockChainWithMetrics) Initialize() error { return c.b.Initialize() } +func (c *blockChainWithMetrics) CreateMempool() (bchain.Mempool, error) { + return c.b.CreateMempool() +} + +func (c *blockChainWithMetrics) InitializeMempool() error { + return c.b.InitializeMempool() +} + func (c *blockChainWithMetrics) Shutdown(ctx context.Context) error { return c.b.Shutdown(ctx) } @@ -197,9 +209,9 @@ func (c *blockChainWithMetrics) GetBlockInfo(hash string) (v *bchain.BlockInfo, return c.b.GetBlockInfo(hash) } -func (c *blockChainWithMetrics) GetMempool() (v []string, err error) { - defer func(s time.Time) { c.observeRPCLatency("GetMempool", s, err) }(time.Now()) - return c.b.GetMempool() +func (c *blockChainWithMetrics) GetMempoolTransactions() (v []string, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactions", s, err) }(time.Now()) + return c.b.GetMempoolTransactions() } func (c *blockChainWithMetrics) GetTransaction(txid string) (v *bchain.Tx, err error) { @@ -232,25 +244,6 @@ func (c *blockChainWithMetrics) SendRawTransaction(tx string) (v string, err err return c.b.SendRawTransaction(tx) } -func (c *blockChainWithMetrics) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (count int, err error) { - defer func(s time.Time) { c.observeRPCLatency("ResyncMempool", s, err) }(time.Now()) - count, err = c.b.ResyncMempool(onNewTxAddr) - if err == nil { - c.m.MempoolSize.Set(float64(count)) - } - return count, err -} - -func (c *blockChainWithMetrics) GetMempoolTransactions(address string) (v []bchain.Outpoint, err error) { - defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactions", s, err) }(time.Now()) - return c.b.GetMempoolTransactions(address) -} - -func (c *blockChainWithMetrics) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []bchain.Outpoint, err error) { - defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactionsForAddrDesc", s, err) }(time.Now()) - return c.b.GetMempoolTransactionsForAddrDesc(addrDesc) -} - func (c *blockChainWithMetrics) GetMempoolEntry(txid string) (v *bchain.MempoolEntry, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempoolEntry", s, err) }(time.Now()) return c.b.GetMempoolEntry(txid) @@ -284,3 +277,35 @@ func (c *blockChainWithMetrics) EthereumTypeGetErc20ContractBalance(addrDesc, co defer func(s time.Time) { c.observeRPCLatency("EthereumTypeGetErc20ContractInfo", s, err) }(time.Now()) return c.b.EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc) } + +type mempoolWithMetrics struct { + mempool bchain.Mempool + m *common.Metrics +} + +func (c *mempoolWithMetrics) observeRPCLatency(method string, start time.Time, err error) { + var e string + if err != nil { + e = err.Error() + } + c.m.RPCLatency.With(common.Labels{"method": method, "error": e}).Observe(float64(time.Since(start)) / 1e6) // in milliseconds +} + +func (c *mempoolWithMetrics) Resync(onNewTxAddr bchain.OnNewTxAddrFunc) (count int, err error) { + defer func(s time.Time) { c.observeRPCLatency("ResyncMempool", s, err) }(time.Now()) + count, err = c.mempool.Resync(onNewTxAddr) + if err == nil { + c.m.MempoolSize.Set(float64(count)) + } + return count, err +} + +func (c *mempoolWithMetrics) GetTransactions(address string) (v []bchain.Outpoint, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactions", s, err) }(time.Now()) + return c.mempool.GetTransactions(address) +} + +func (c *mempoolWithMetrics) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor) (v []bchain.Outpoint, err error) { + defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactionsForAddrDesc", s, err) }(time.Now()) + return c.mempool.GetAddrDescTransactions(addrDesc) +} diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index ef7574cd..f26830d2 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -101,37 +101,15 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT return s, nil } -// GetChainInfoAndInitializeMempool is called by Initialize and reused by other coins -// it contacts the blockchain rpc interface for the first time -// and if successful it connects to ZeroMQ and creates mempool handler -func (b *BitcoinRPC) GetChainInfoAndInitializeMempool(bc bchain.BlockChain) (string, error) { - // try to connect to block chain and get some info - ci, err := bc.GetChainInfo() - if err != nil { - return "", err - } - chainName := ci.Chain - - mq, err := bchain.NewMQ(b.ChainConfig.MessageQueueBinding, b.pushHandler) - if err != nil { - glog.Error("mq: ", err) - return "", err - } - b.mq = mq - - b.Mempool = bchain.NewMempoolBitcoinType(bc, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers) - - return chainName, nil -} - // Initialize initializes BitcoinRPC instance. func (b *BitcoinRPC) Initialize() error { b.ChainConfig.SupportsEstimateFee = false - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) @@ -152,6 +130,28 @@ func (b *BitcoinRPC) Initialize() error { return nil } +// CreateMempool creates mempool if not already created, however does not initialize it +func (b *BitcoinRPC) CreateMempool() (bchain.Mempool, error) { + if b.Mempool == nil { + b.Mempool = bchain.NewMempoolBitcoinType(b, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers) + } + return b.Mempool, nil +} + +// InitializeMempool creates ZeroMQ subscription +func (b *BitcoinRPC) InitializeMempool() error { + if b.mq == nil { + mq, err := bchain.NewMQ(b.ChainConfig.MessageQueueBinding, b.pushHandler) + if err != nil { + glog.Error("mq: ", err) + return err + } + b.mq = mq + } + return nil +} + +// Shutdown ZeroMQ and other resources func (b *BitcoinRPC) Shutdown(ctx context.Context) error { if b.mq != nil { if err := b.mq.Shutdown(ctx); err != nil { @@ -162,10 +162,12 @@ func (b *BitcoinRPC) Shutdown(ctx context.Context) error { return nil } +// GetCoinName returns the coin name func (b *BitcoinRPC) GetCoinName() string { return b.ChainConfig.CoinName } +// GetSubversion returns the backend subversion func (b *BitcoinRPC) GetSubversion() string { return b.ChainConfig.Subversion } @@ -458,6 +460,7 @@ func (b *BitcoinRPC) GetChainInfo() (*bchain.ChainInfo, error) { return rv, nil } +// IsErrBlockNotFound returns true if error means block was not found func IsErrBlockNotFound(err *bchain.RPCError) bool { return err.Message == "Block not found" || err.Message == "Block height out of range" @@ -634,8 +637,8 @@ func (b *BitcoinRPC) GetBlockFull(hash string) (*bchain.Block, error) { return &res.Result, nil } -// GetMempool returns transactions in mempool -func (b *BitcoinRPC) GetMempool() ([]string, error) { +// GetMempoolTransactions returns transactions in mempool +func (b *BitcoinRPC) GetMempoolTransactions() ([]string, error) { glog.V(1).Info("rpc: getrawmempool") res := ResGetMempool{} @@ -651,6 +654,7 @@ func (b *BitcoinRPC) GetMempool() ([]string, error) { return res.Result, nil } +// IsMissingTx return true if error means missing tx func IsMissingTx(err *bchain.RPCError) bool { if err.Code == -5 { // "No such mempool or blockchain transaction" return true @@ -732,23 +736,6 @@ func (b *BitcoinRPC) getRawTransaction(txid string) (json.RawMessage, error) { return res.Result, nil } -// ResyncMempool gets mempool transactions and maps output scripts to transactions. -// ResyncMempool is not reentrant, it should be called from a single thread. -// Return value is number of transactions in mempool -func (b *BitcoinRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { - return b.Mempool.Resync(onNewTxAddr) -} - -// GetMempoolTransactions returns slice of mempool transactions for given address -func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]bchain.Outpoint, error) { - return b.Mempool.GetTransactions(address) -} - -// GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor -func (b *BitcoinRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]bchain.Outpoint, error) { - return b.Mempool.GetAddrDescTransactions(addrDesc) -} - // EstimateSmartFee returns fee estimation func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { // use EstimateFee if EstimateSmartFee is not supported @@ -810,7 +797,7 @@ func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { return r, nil } -// SendRawTransaction sends raw transaction. +// SendRawTransaction sends raw transaction func (b *BitcoinRPC) SendRawTransaction(tx string) (string, error) { glog.V(1).Info("rpc: sendrawtransaction") diff --git a/bchain/coins/btg/bgoldrpc.go b/bchain/coins/btg/bgoldrpc.go index b158ba32..a590de97 100644 --- a/bchain/coins/btg/bgoldrpc.go +++ b/bchain/coins/btg/bgoldrpc.go @@ -29,10 +29,11 @@ func NewBGoldRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp // Initialize initializes BGoldRPC instance. func (b *BGoldRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/dash/dashrpc.go b/bchain/coins/dash/dashrpc.go index 66629d79..2320113d 100644 --- a/bchain/coins/dash/dashrpc.go +++ b/bchain/coins/dash/dashrpc.go @@ -34,10 +34,11 @@ func NewDashRPC(config json.RawMessage, pushHandler func(bchain.NotificationType // Initialize initializes DashRPC instance. func (b *DashRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/digibyte/digibyterpc.go b/bchain/coins/digibyte/digibyterpc.go index 0cc47e9e..d0f6f32c 100644 --- a/bchain/coins/digibyte/digibyterpc.go +++ b/bchain/coins/digibyte/digibyterpc.go @@ -31,10 +31,11 @@ func NewDigiByteRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes DigiByteRPC instance. func (b *DigiByteRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/dogecoin/dogecoinrpc.go b/bchain/coins/dogecoin/dogecoinrpc.go index 1fc21618..9d6b4e63 100644 --- a/bchain/coins/dogecoin/dogecoinrpc.go +++ b/bchain/coins/dogecoin/dogecoinrpc.go @@ -31,10 +31,11 @@ func NewDogecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes DogecoinRPC instance. func (b *DogecoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 2a34c077..bac05d97 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -160,11 +160,24 @@ func (b *EthereumRPC) Initialize() error { } glog.Info("rpc: block chain ", b.Network) + return nil +} + +// CreateMempool creates mempool if not already created, however does not initialize it +func (b *EthereumRPC) CreateMempool() (bchain.Mempool, error) { + if b.Mempool == nil { + b.Mempool = bchain.NewMempoolEthereumType(b) + } + return b.Mempool, nil +} + +// InitializeMempool creates subscriptions to newHeads and newPendingTransactions +func (b *EthereumRPC) InitializeMempool() error { if b.isETC { glog.Info(b.ChainConfig.CoinName, " does not support subscription to newHeads") } else { // subscriptions - if err = b.subscribe(func() (*rpc.ClientSubscription, error) { + if err := b.subscribe(func() (*rpc.ClientSubscription, error) { // invalidate the previous subscription - it is either the first one or there was an error b.newBlockSubscription = nil ctx, cancel := context.WithTimeout(context.Background(), b.timeout) @@ -180,7 +193,8 @@ func (b *EthereumRPC) Initialize() error { return err } } - if err = b.subscribe(func() (*rpc.ClientSubscription, error) { + + if err := b.subscribe(func() (*rpc.ClientSubscription, error) { // invalidate the previous subscription - it is either the first one or there was an error b.newTxSubscription = nil ctx, cancel := context.WithTimeout(context.Background(), b.timeout) @@ -195,10 +209,6 @@ func (b *EthereumRPC) Initialize() error { }); err != nil { return err } - - // create mempool - b.Mempool = bchain.NewMempoolEthereumType(b) - return nil } @@ -628,8 +638,8 @@ func (b *EthereumRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, er return json.RawMessage(m), err } -// GetMempool returns transactions in mempool -func (b *EthereumRPC) GetMempool() ([]string, error) { +// GetMempoolTransactions returns transactions in mempool +func (b *EthereumRPC) GetMempoolTransactions() ([]string, error) { raw, err := b.getBlockRaw("pending", 0, false) if err != nil { return nil, err @@ -737,28 +747,6 @@ func (b *EthereumRPC) EthereumTypeGetNonce(addrDesc bchain.AddressDescriptor) (u return b.client.NonceAt(ctx, ethcommon.BytesToAddress(addrDesc), nil) } -// ResyncMempool gets mempool transactions and maps output scripts to transactions. -// ResyncMempool is not reentrant, it should be called from a single thread. -// Return value is number of transactions in mempool -func (b *EthereumRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { - return b.Mempool.Resync(onNewTxAddr) -} - -// GetMempoolTransactions returns slice of mempool transactions for given address -func (b *EthereumRPC) GetMempoolTransactions(address string) ([]bchain.Outpoint, error) { - return b.Mempool.GetTransactions(address) -} - -// GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor -func (b *EthereumRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]bchain.Outpoint, error) { - return b.Mempool.GetAddrDescTransactions(addrDesc) -} - -// GetMempoolEntry is not supported by etherem -func (b *EthereumRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { - return nil, errors.New("GetMempoolEntry: not supported") -} - // GetChainParser returns ethereum BlockChainParser func (b *EthereumRPC) GetChainParser() bchain.BlockChainParser { return b.Parser diff --git a/bchain/coins/flo/florpc.go b/bchain/coins/flo/florpc.go index 94f831bb..42549034 100644 --- a/bchain/coins/flo/florpc.go +++ b/bchain/coins/flo/florpc.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/json" + "github.com/juju/errors" "github.com/golang/glog" @@ -32,10 +33,11 @@ func NewFloRPC(config json.RawMessage, pushHandler func(bchain.NotificationType) // Initialize initializes FloRPC instance. func (f *FloRPC) Initialize() error { - chainName, err := f.GetChainInfoAndInitializeMempool(f) + ci, err := f.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/fujicoin/fujicoinrpc.go b/bchain/coins/fujicoin/fujicoinrpc.go index 16c562cc..d7588812 100644 --- a/bchain/coins/fujicoin/fujicoinrpc.go +++ b/bchain/coins/fujicoin/fujicoinrpc.go @@ -31,10 +31,11 @@ func NewFujicoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes FujicoinRPC instance. func (b *FujicoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/gamecredits/gamecreditsrpc.go b/bchain/coins/gamecredits/gamecreditsrpc.go index bb3cecb0..763ff018 100644 --- a/bchain/coins/gamecredits/gamecreditsrpc.go +++ b/bchain/coins/gamecredits/gamecreditsrpc.go @@ -31,10 +31,11 @@ func NewGameCreditsRPC(config json.RawMessage, pushHandler func(bchain.Notificat // Initialize initializes GameCreditsRPC instance. func (b *GameCreditsRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/grs/grsrpc.go b/bchain/coins/grs/grsrpc.go index d3afb9f0..8d4495fa 100644 --- a/bchain/coins/grs/grsrpc.go +++ b/bchain/coins/grs/grsrpc.go @@ -27,10 +27,11 @@ func NewGroestlcoinRPC(config json.RawMessage, pushHandler func(bchain.Notificat // Initialize initializes GroestlcoinRPC instance. func (g *GroestlcoinRPC) Initialize() error { - chainName, err := g.GetChainInfoAndInitializeMempool(g) + ci, err := g.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/koto/kotorpc.go b/bchain/coins/koto/kotorpc.go index 7bb35868..5dc87295 100644 --- a/bchain/coins/koto/kotorpc.go +++ b/bchain/coins/koto/kotorpc.go @@ -28,10 +28,11 @@ func NewKotoRPC(config json.RawMessage, pushHandler func(bchain.NotificationType // Initialize initializes KotoRPC instance. func (z *KotoRPC) Initialize() error { - chainName, err := z.GetChainInfoAndInitializeMempool(z) + ci, err := z.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/liquid/liquidrpc.go b/bchain/coins/liquid/liquidrpc.go index 7bfc54a7..1ec45fb2 100644 --- a/bchain/coins/liquid/liquidrpc.go +++ b/bchain/coins/liquid/liquidrpc.go @@ -31,10 +31,11 @@ func NewLiquidRPC(config json.RawMessage, pushHandler func(bchain.NotificationTy // Initialize initializes GameCreditsRPC instance. func (b *LiquidRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/litecoin/litecoinrpc.go b/bchain/coins/litecoin/litecoinrpc.go index e7345251..274e3054 100644 --- a/bchain/coins/litecoin/litecoinrpc.go +++ b/bchain/coins/litecoin/litecoinrpc.go @@ -31,10 +31,11 @@ func NewLitecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes LitecoinRPC instance. func (b *LitecoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/monacoin/monacoinrpc.go b/bchain/coins/monacoin/monacoinrpc.go index dacddb4c..77accd43 100644 --- a/bchain/coins/monacoin/monacoinrpc.go +++ b/bchain/coins/monacoin/monacoinrpc.go @@ -31,10 +31,11 @@ func NewMonacoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes MonacoinRPC instance. func (b *MonacoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/myriad/myriadrpc.go b/bchain/coins/myriad/myriadrpc.go index f0bd201a..cafa00b6 100644 --- a/bchain/coins/myriad/myriadrpc.go +++ b/bchain/coins/myriad/myriadrpc.go @@ -31,10 +31,11 @@ func NewMyriadRPC(config json.RawMessage, pushHandler func(bchain.NotificationTy // Initialize initializes MyriadRPC instance. func (b *MyriadRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/namecoin/namecoinrpc.go b/bchain/coins/namecoin/namecoinrpc.go index 89dc245c..927c095d 100644 --- a/bchain/coins/namecoin/namecoinrpc.go +++ b/bchain/coins/namecoin/namecoinrpc.go @@ -31,10 +31,11 @@ func NewNamecoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes NamecoinRPC instance. func (b *NamecoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/pivx/pivxrpc.go b/bchain/coins/pivx/pivxrpc.go index 8979ee4f..38f2c2cc 100644 --- a/bchain/coins/pivx/pivxrpc.go +++ b/bchain/coins/pivx/pivxrpc.go @@ -32,10 +32,11 @@ func NewPivXRPC(config json.RawMessage, pushHandler func(bchain.NotificationType // Initialize initializes PivXRPC instance. func (b *PivXRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/qtum/qtumrpc.go b/bchain/coins/qtum/qtumrpc.go index b5a8cafc..a942ce3b 100644 --- a/bchain/coins/qtum/qtumrpc.go +++ b/bchain/coins/qtum/qtumrpc.go @@ -4,6 +4,7 @@ import ( "blockbook/bchain" "blockbook/bchain/coins/btc" "encoding/json" + "github.com/golang/glog" ) @@ -30,10 +31,11 @@ func NewQtumRPC(config json.RawMessage, pushHandler func(bchain.NotificationType // Initialize initializes QtumRPC instance. func (b *QtumRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/vertcoin/vertcoinrpc.go b/bchain/coins/vertcoin/vertcoinrpc.go index f282dc78..ec1e1ba2 100644 --- a/bchain/coins/vertcoin/vertcoinrpc.go +++ b/bchain/coins/vertcoin/vertcoinrpc.go @@ -31,10 +31,11 @@ func NewVertcoinRPC(config json.RawMessage, pushHandler func(bchain.Notification // Initialize initializes VertcoinRPC instance. func (b *VertcoinRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain glog.Info("Chain name ", chainName) params := GetChainParams(chainName) diff --git a/bchain/coins/xzc/zcoinrpc.go b/bchain/coins/xzc/zcoinrpc.go index 1117cbc3..22dfbe92 100644 --- a/bchain/coins/xzc/zcoinrpc.go +++ b/bchain/coins/xzc/zcoinrpc.go @@ -36,10 +36,11 @@ func NewZcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp } func (zc *ZcoinRPC) Initialize() error { - chainName, err := zc.GetChainInfoAndInitializeMempool(zc) + ci, err := zc.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/coins/zec/zcashrpc.go b/bchain/coins/zec/zcashrpc.go index 3ecffea3..4da5f8d4 100644 --- a/bchain/coins/zec/zcashrpc.go +++ b/bchain/coins/zec/zcashrpc.go @@ -30,10 +30,11 @@ func NewZCashRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp // Initialize initializes ZCashRPC instance func (z *ZCashRPC) Initialize() error { - chainName, err := z.GetChainInfoAndInitializeMempool(z) + ci, err := z.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index db12ffd4..e87cc937 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -163,7 +163,7 @@ func (m *MempoolBitcoinType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { start := time.Now() glog.V(1).Info("mempool: resync") m.onNewTxAddr = onNewTxAddr - txs, err := m.chain.GetMempool() + txs, err := m.chain.GetMempoolTransactions() if err != nil { return 0, err } diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index eff4bc53..1f05c9c5 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -62,7 +62,7 @@ func appendAddress(io []addrIndex, i int32, a string, parser BlockChainParser) [ func (m *MempoolEthereumType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { start := time.Now() glog.V(1).Info("Mempool: resync") - txs, err := m.chain.GetMempool() + txs, err := m.chain.GetMempoolTransactions() if err != nil { return 0, err } diff --git a/bchain/types.go b/bchain/types.go index ead62535..044122e6 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -194,7 +194,13 @@ type OnNewTxAddrFunc func(tx *Tx, desc AddressDescriptor) // BlockChain defines common interface to block chain daemon type BlockChain interface { // life-cycle methods + // intialize the block chain connector Initialize() error + // create mempool but do not initialize it + CreateMempool() (Mempool, error) + // initialize mempool, create ZeroMQ (or other) subscription + InitializeMempool() error + // shutdown mempool, ZeroMQ and block chain connections Shutdown(ctx context.Context) error // chain info IsTestnet() bool @@ -209,17 +215,13 @@ type BlockChain interface { GetBlockHeader(hash string) (*BlockHeader, error) GetBlock(hash string, height uint32) (*Block, error) GetBlockInfo(hash string) (*BlockInfo, error) - GetMempool() ([]string, error) + GetMempoolTransactions() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) GetTransactionSpecific(tx *Tx) (json.RawMessage, error) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) - // mempool - ResyncMempool(onNewTxAddr OnNewTxAddrFunc) (int, error) - GetMempoolTransactions(address string) ([]Outpoint, error) - GetMempoolTransactionsForAddrDesc(addrDesc AddressDescriptor) ([]Outpoint, error) GetMempoolEntry(txid string) (*MempoolEntry, error) // parser GetChainParser() BlockChainParser @@ -270,3 +272,10 @@ type BlockChainParser interface { // EthereumType specific EthereumTypeGetErc20FromTx(tx *Tx) ([]Erc20Transfer, error) } + +// Mempool defines common interface to mempool +type Mempool interface { + Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) + GetTransactions(address string) ([]Outpoint, error) + GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) +} diff --git a/blockbook.go b/blockbook.go index 44d8def9..6315ad52 100644 --- a/blockbook.go +++ b/blockbook.go @@ -82,6 +82,7 @@ var ( chanSyncMempoolDone = make(chan struct{}) chanStoreInternalStateDone = make(chan struct{}) chain bchain.BlockChain + mempool bchain.Mempool index *db.RocksDB txCache *db.TxCache metrics *common.Metrics @@ -98,26 +99,27 @@ func init() { glog.CopyStandardLogTo("INFO") } -func getBlockChainWithRetry(coin string, configfile string, pushHandler func(bchain.NotificationType), metrics *common.Metrics, seconds int) (bchain.BlockChain, error) { +func getBlockChainWithRetry(coin string, configfile string, pushHandler func(bchain.NotificationType), metrics *common.Metrics, seconds int) (bchain.BlockChain, bchain.Mempool, error) { var chain bchain.BlockChain + var mempool bchain.Mempool var err error timer := time.NewTimer(time.Second) for i := 0; ; i++ { - if chain, err = coins.NewBlockChain(coin, configfile, pushHandler, metrics); err != nil { + if chain, mempool, err = coins.NewBlockChain(coin, configfile, pushHandler, metrics); err != nil { if i < seconds { glog.Error("rpc: ", err, " Retrying...") select { case <-chanOsSignal: - return nil, errors.New("Interrupted") + return nil, nil, errors.New("Interrupted") case <-timer.C: timer.Reset(time.Second) continue } } else { - return nil, err + return nil, nil, err } } - return chain, nil + return chain, mempool, nil } } @@ -162,7 +164,7 @@ func main() { glog.Fatal("metrics: ", err) } - if chain, err = getBlockChainWithRetry(coin, *blockchain, pushSynchronizationHandler, metrics, 60); err != nil { + if chain, mempool, err = getBlockChainWithRetry(coin, *blockchain, pushSynchronizationHandler, metrics, 60); err != nil { glog.Fatal("rpc: ", err) } @@ -209,29 +211,7 @@ func main() { } if *rollbackHeight >= 0 { - bestHeight, bestHash, err := index.GetBestBlock() - if err != nil { - glog.Error("rollbackHeight: ", err) - return - } - if uint32(*rollbackHeight) > bestHeight { - glog.Infof("nothing to rollback, rollbackHeight %d, bestHeight: %d", *rollbackHeight, bestHeight) - } else { - hashes := []string{bestHash} - for height := bestHeight - 1; height >= uint32(*rollbackHeight); height-- { - hash, err := index.GetBlockHash(height) - if err != nil { - glog.Error("rollbackHeight: ", err) - return - } - hashes = append(hashes, hash) - } - err = syncWorker.DisconnectBlocks(uint32(*rollbackHeight), bestHeight, hashes) - if err != nil { - glog.Error("rollbackHeight: ", err) - return - } - } + performRollback() return } @@ -247,7 +227,7 @@ func main() { var internalServer *server.InternalServer if *internalBinding != "" { - internalServer, err = server.NewInternalServer(*internalBinding, *certFiles, index, chain, txCache, internalState) + internalServer, err = server.NewInternalServer(*internalBinding, *certFiles, index, chain, mempool, txCache, internalState) if err != nil { glog.Error("https: ", err) return @@ -268,7 +248,7 @@ func main() { var publicServer *server.PublicServer if *publicBinding != "" { // start public server in limited functionality, extend it after sync is finished by calling ConnectFullPublicInterface - publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, txCache, *explorerURL, metrics, internalState, *debugMode) + publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, mempool, txCache, *explorerURL, metrics, internalState, *debugMode) if err != nil { glog.Error("socketio: ", err) return @@ -295,8 +275,14 @@ func main() { glog.Error("resyncIndex ", err) return } + // initialize mempool after the initial sync is complete + err = chain.InitializeMempool() + if err != nil { + glog.Error("initializeMempool ", err) + return + } var mempoolCount int - if mempoolCount, err = chain.ResyncMempool(nil); err != nil { + if mempoolCount, err = mempool.Resync(nil); err != nil { glog.Error("resyncMempool ", err) return } @@ -341,8 +327,34 @@ func main() { } } +func performRollback() { + bestHeight, bestHash, err := index.GetBestBlock() + if err != nil { + glog.Error("rollbackHeight: ", err) + return + } + if uint32(*rollbackHeight) > bestHeight { + glog.Infof("nothing to rollback, rollbackHeight %d, bestHeight: %d", *rollbackHeight, bestHeight) + } else { + hashes := []string{bestHash} + for height := bestHeight - 1; height >= uint32(*rollbackHeight); height-- { + hash, err := index.GetBlockHash(height) + if err != nil { + glog.Error("rollbackHeight: ", err) + return + } + hashes = append(hashes, hash) + } + err = syncWorker.DisconnectBlocks(uint32(*rollbackHeight), bestHeight, hashes) + if err != nil { + glog.Error("rollbackHeight: ", err) + return + } + } +} + func blockbookAppInfoMetric(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState, metrics *common.Metrics) error { - api, err := api.NewWorker(db, chain, txCache, is) + api, err := api.NewWorker(db, chain, mempool, txCache, is) if err != nil { return err } @@ -442,7 +454,7 @@ func syncMempoolLoop() { // resync mempool about every minute if there are no chanSyncMempool requests, with debounce 1 second tickAndDebounce(time.Duration(*resyncMempoolPeriodMs)*time.Millisecond, debounceResyncMempoolMs*time.Millisecond, chanSyncMempool, func() { internalState.StartedMempoolSync() - if count, err := chain.ResyncMempool(onNewTxAddr); err != nil { + if count, err := mempool.Resync(onNewTxAddr); err != nil { glog.Error("syncMempoolLoop ", errors.ErrorStack(err)) } else { internalState.FinishedMempoolSync(count) diff --git a/server/internal.go b/server/internal.go index 6ef09f1e..1f7c3b89 100644 --- a/server/internal.go +++ b/server/internal.go @@ -23,13 +23,14 @@ type InternalServer struct { txCache *db.TxCache chain bchain.BlockChain chainParser bchain.BlockChainParser + mempool bchain.Mempool is *common.InternalState api *api.Worker } // NewInternalServer creates new internal http interface to blockbook and returns its handle -func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is *common.InternalState) (*InternalServer, error) { - api, err := api.NewWorker(db, chain, txCache, is) +func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, txCache *db.TxCache, is *common.InternalState) (*InternalServer, error) { + api, err := api.NewWorker(db, chain, mempool, txCache, is) if err != nil { return nil, err } @@ -47,6 +48,7 @@ func NewInternalServer(binding, certFiles string, db *db.RocksDB, chain bchain.B txCache: txCache, chain: chain, chainParser: chain.GetChainParser(), + mempool: mempool, is: is, api: api, } diff --git a/server/public.go b/server/public.go index 12deb354..7543ec2a 100644 --- a/server/public.go +++ b/server/public.go @@ -45,6 +45,7 @@ type PublicServer struct { txCache *db.TxCache chain bchain.BlockChain chainParser bchain.BlockChainParser + mempool bchain.Mempool api *api.Worker explorerURL string internalExplorer bool @@ -56,19 +57,19 @@ type PublicServer struct { // NewPublicServer creates new public server http interface to blockbook and returns its handle // only basic functionality is mapped, to map all functions, call -func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState, debugMode bool) (*PublicServer, error) { +func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, txCache *db.TxCache, explorerURL string, metrics *common.Metrics, is *common.InternalState, debugMode bool) (*PublicServer, error) { - api, err := api.NewWorker(db, chain, txCache, is) + api, err := api.NewWorker(db, chain, mempool, txCache, is) if err != nil { return nil, err } - socketio, err := NewSocketIoServer(db, chain, txCache, metrics, is) + socketio, err := NewSocketIoServer(db, chain, mempool, txCache, metrics, is) if err != nil { return nil, err } - websocket, err := NewWebsocketServer(db, chain, txCache, metrics, is) + websocket, err := NewWebsocketServer(db, chain, mempool, txCache, metrics, is) if err != nil { return nil, err } @@ -91,6 +92,7 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch txCache: txCache, chain: chain, chainParser: chain.GetChainParser(), + mempool: mempool, explorerURL: explorerURL, internalExplorer: explorerURL == "", metrics: metrics, diff --git a/server/public_test.go b/server/public_test.go index e203d947..aa64044e 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -85,6 +85,11 @@ func setupPublicHTTPServer(t *testing.T) (*PublicServer, string) { glog.Fatal("fakechain: ", err) } + mempool, err := chain.CreateMempool() + if err != nil { + glog.Fatal("mempool: ", err) + } + // caching is switched off because test transactions do not have hex data txCache, err := db.NewTxCache(d, chain, metrics, is, false) if err != nil { @@ -92,7 +97,7 @@ func setupPublicHTTPServer(t *testing.T) (*PublicServer, string) { } // s.Run is never called, binding can be to any port - s, err := NewPublicServer("localhost:12345", "", d, chain, txCache, "", metrics, is, false) + s, err := NewPublicServer("localhost:12345", "", d, chain, mempool, txCache, "", metrics, is, false) if err != nil { t.Fatal(err) } diff --git a/server/socketio.go b/server/socketio.go index fc70510d..a1790762 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -26,14 +26,15 @@ type SocketIoServer struct { txCache *db.TxCache chain bchain.BlockChain chainParser bchain.BlockChainParser + mempool bchain.Mempool metrics *common.Metrics is *common.InternalState api *api.Worker } // NewSocketIoServer creates new SocketIo interface to blockbook and returns its handle -func NewSocketIoServer(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, metrics *common.Metrics, is *common.InternalState) (*SocketIoServer, error) { - api, err := api.NewWorker(db, chain, txCache, is) +func NewSocketIoServer(db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, txCache *db.TxCache, metrics *common.Metrics, is *common.InternalState) (*SocketIoServer, error) { + api, err := api.NewWorker(db, chain, mempool, txCache, is) if err != nil { return nil, err } @@ -64,6 +65,7 @@ func NewSocketIoServer(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCa txCache: txCache, chain: chain, chainParser: chain.GetChainParser(), + mempool: mempool, metrics: metrics, is: is, api: api, @@ -224,7 +226,7 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res return res, err } } else { - o, err := s.chain.GetMempoolTransactions(address) + o, err := s.mempool.GetTransactions(address) if err != nil { return res, err } diff --git a/server/websocket.go b/server/websocket.go index 0c9ec060..3f504ea3 100644 --- a/server/websocket.go +++ b/server/websocket.go @@ -59,6 +59,7 @@ type WebsocketServer struct { txCache *db.TxCache chain bchain.BlockChain chainParser bchain.BlockChainParser + mempool bchain.Mempool metrics *common.Metrics is *common.InternalState api *api.Worker @@ -70,8 +71,8 @@ type WebsocketServer struct { } // NewWebsocketServer creates new websocket interface to blockbook and returns its handle -func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, metrics *common.Metrics, is *common.InternalState) (*WebsocketServer, error) { - api, err := api.NewWorker(db, chain, txCache, is) +func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, mempool bchain.Mempool, txCache *db.TxCache, metrics *common.Metrics, is *common.InternalState) (*WebsocketServer, error) { + api, err := api.NewWorker(db, chain, mempool, txCache, is) if err != nil { return nil, err } @@ -89,6 +90,7 @@ func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxC txCache: txCache, chain: chain, chainParser: chain.GetChainParser(), + mempool: mempool, metrics: metrics, is: is, api: api, diff --git a/tests/dbtestdata/fakechain.go b/tests/dbtestdata/fakechain.go index 71515bbb..2f05bd4b 100644 --- a/tests/dbtestdata/fakechain.go +++ b/tests/dbtestdata/fakechain.go @@ -17,10 +17,18 @@ func NewFakeBlockChain(parser bchain.BlockChainParser) (bchain.BlockChain, error return &fakeBlockChain{&bchain.BaseChain{Parser: parser}}, nil } +func (b *fakeBlockChain) CreateMempool() (bchain.Mempool, error) { + return bchain.NewMempoolBitcoinType(b, 1, 1), nil +} + func (c *fakeBlockChain) Initialize() error { return nil } +func (c *fakeBlockChain) InitializeMempool() error { + return nil +} + func (c *fakeBlockChain) Shutdown(ctx context.Context) error { return nil } @@ -118,10 +126,6 @@ func (c *fakeBlockChain) GetBlockInfo(hash string) (v *bchain.BlockInfo, err err return nil, bchain.ErrBlockNotFound } -func (c *fakeBlockChain) GetMempool() (v []string, err error) { - return nil, errors.New("Not implemented") -} - func getTxInBlock(b *bchain.Block, txid string) *bchain.Tx { for _, tx := range b.Txs { if tx.Txid == txid { @@ -179,22 +183,12 @@ func (c *fakeBlockChain) SendRawTransaction(tx string) (v string, err error) { return "", errors.New("Invalid data") } -func (c *fakeBlockChain) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (count int, err error) { - return 0, errors.New("Not implemented") -} - -func (c *fakeBlockChain) GetMempoolTransactions(address string) (v []bchain.Outpoint, err error) { - return nil, errors.New("Not implemented") -} - -func (c *fakeBlockChain) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []bchain.Outpoint, err error) { - return []bchain.Outpoint{}, nil -} - -func (c *fakeBlockChain) GetMempoolEntry(txid string) (v *bchain.MempoolEntry, err error) { - return nil, errors.New("Not implemented") -} - +// GetChainParser returns parser for the blockchain func (c *fakeBlockChain) GetChainParser() bchain.BlockChainParser { return c.Parser } + +// GetMempoolTransactions returns transactions in mempool +func (b *fakeBlockChain) GetMempoolTransactions() ([]string, error) { + return nil, errors.New("Not implemented") +} diff --git a/tests/integration.go b/tests/integration.go index ad0815f4..789f71fc 100644 --- a/tests/integration.go +++ b/tests/integration.go @@ -5,7 +5,7 @@ package tests import ( "blockbook/bchain" "blockbook/bchain/coins" - "blockbook/build/tools" + build "blockbook/build/tools" "blockbook/tests/rpc" "blockbook/tests/sync" "encoding/json" @@ -23,7 +23,7 @@ import ( "github.com/martinboehm/btcutil/chaincfg" ) -type TestFunc func(t *testing.T, coin string, chain bchain.BlockChain, testConfig json.RawMessage) +type TestFunc func(t *testing.T, coin string, chain bchain.BlockChain, mempool bchain.Mempool, testConfig json.RawMessage) var integrationTests = map[string]TestFunc{ "rpc": rpc.IntegrationTest, @@ -76,7 +76,7 @@ func runTests(t *testing.T, coin string, cfg map[string]json.RawMessage) { } defer chaincfg.ResetParams() - bc, err := makeBlockChain(coin) + bc, m, err := makeBlockChain(coin) if err != nil { if err == notConnectedError { t.Fatal(err) @@ -86,44 +86,44 @@ func runTests(t *testing.T, coin string, cfg map[string]json.RawMessage) { for test, c := range cfg { if fn, found := integrationTests[test]; found { - t.Run(test, func(t *testing.T) { fn(t, coin, bc, c) }) + t.Run(test, func(t *testing.T) { fn(t, coin, bc, m, c) }) } else { t.Errorf("Test not found: %s", test) } } } -func makeBlockChain(coin string) (bchain.BlockChain, error) { +func makeBlockChain(coin string) (bchain.BlockChain, bchain.Mempool, error) { c, err := build.LoadConfig("../configs", coin) if err != nil { - return nil, err + return nil, nil, err } outputDir, err := ioutil.TempDir("", "integration_test") if err != nil { - return nil, err + return nil, nil, err } defer os.RemoveAll(outputDir) err = build.GeneratePackageDefinitions(c, "../build/templates", outputDir) if err != nil { - return nil, err + return nil, nil, err } b, err := ioutil.ReadFile(filepath.Join(outputDir, "blockbook", "blockchaincfg.json")) if err != nil { - return nil, err + return nil, nil, err } var cfg json.RawMessage err = json.Unmarshal(b, &cfg) if err != nil { - return nil, err + return nil, nil, err } coinName, err := getName(cfg) if err != nil { - return nil, err + return nil, nil, err } return initBlockChain(coinName, cfg) @@ -147,29 +147,39 @@ func getName(raw json.RawMessage) (string, error) { } } -func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, error) { +func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, bchain.Mempool, error) { factory, found := coins.BlockChainFactories[coinName] if !found { - return nil, fmt.Errorf("Factory function not found") + return nil, nil, fmt.Errorf("Factory function not found") } cli, err := factory(cfg, func(_ bchain.NotificationType) {}) if err != nil { if isNetError(err) { - return nil, notConnectedError + return nil, nil, notConnectedError } - return nil, fmt.Errorf("Factory function failed: %s", err) + return nil, nil, fmt.Errorf("Factory function failed: %s", err) } err = cli.Initialize() if err != nil { if isNetError(err) { - return nil, notConnectedError + return nil, nil, notConnectedError } - return nil, fmt.Errorf("BlockChain initialization failed: %s", err) + return nil, nil, fmt.Errorf("BlockChain initialization failed: %s", err) } - return cli, nil + mempool, err := cli.CreateMempool() + if err != nil { + return nil, nil, fmt.Errorf("Mempool creation failed: %s", err) + } + + err = cli.InitializeMempool() + if err != nil { + return nil, nil, fmt.Errorf("Mempool initialization failed: %s", err) + } + + return cli, mempool, nil } func isNetError(err error) bool { diff --git a/tests/rpc/rpc.go b/tests/rpc/rpc.go index e8722062..c4122d3f 100644 --- a/tests/rpc/rpc.go +++ b/tests/rpc/rpc.go @@ -30,6 +30,7 @@ var testMap = map[string]func(t *testing.T, th *TestHandler){ type TestHandler struct { Chain bchain.BlockChain + Mempool bchain.Mempool TestData *TestData } @@ -42,7 +43,7 @@ type TestData struct { TxDetails map[string]*bchain.Tx `json:"txDetails"` } -func IntegrationTest(t *testing.T, coin string, chain bchain.BlockChain, testConfig json.RawMessage) { +func IntegrationTest(t *testing.T, coin string, chain bchain.BlockChain, mempool bchain.Mempool, testConfig json.RawMessage) { tests, err := getTests(testConfig) if err != nil { t.Fatalf("Failed loading of test list: %s", err) @@ -54,7 +55,11 @@ func IntegrationTest(t *testing.T, coin string, chain bchain.BlockChain, testCon t.Fatalf("Failed loading of test data: %s", err) } - h := TestHandler{Chain: chain, TestData: td} + h := TestHandler{ + Chain: chain, + Mempool: mempool, + TestData: td, + } for _, test := range tests { if f, found := testMap[test]; found { @@ -195,7 +200,7 @@ func testMempoolSync(t *testing.T, h *TestHandler) { for i := 0; i < 3; i++ { txs := getMempool(t, h) - n, err := h.Chain.ResyncMempool(nil) + n, err := h.Mempool.Resync(nil) if err != nil { t.Fatal(err) } @@ -217,7 +222,7 @@ func testMempoolSync(t *testing.T, h *TestHandler) { for txid, addrs := range txid2addrs { for _, a := range addrs { - got, err := h.Chain.GetMempoolTransactions(a) + got, err := h.Mempool.GetTransactions(a) if err != nil { t.Fatalf("address %q: %s", a, err) } @@ -337,7 +342,7 @@ func testGetBlockHeader(t *testing.T, h *TestHandler) { } func getMempool(t *testing.T, h *TestHandler) []string { - txs, err := h.Chain.GetMempool() + txs, err := h.Chain.GetMempoolTransactions() if err != nil { t.Fatal(err) } diff --git a/tests/sync/sync.go b/tests/sync/sync.go index 6c01bfc0..2364da44 100644 --- a/tests/sync/sync.go +++ b/tests/sync/sync.go @@ -54,7 +54,7 @@ type BlockInfo struct { TxDetails []*bchain.Tx `json:"txDetails"` } -func IntegrationTest(t *testing.T, coin string, chain bchain.BlockChain, testConfig json.RawMessage) { +func IntegrationTest(t *testing.T, coin string, chain bchain.BlockChain, mempool bchain.Mempool, testConfig json.RawMessage) { tests, err := getTests(testConfig) if err != nil { t.Fatalf("Failed loading of test list: %s", err) From d734c7d4895eda403d54222d39e4c26bdd3902ae Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 26 Mar 2019 13:04:30 +0100 Subject: [PATCH 02/39] Update xpub segwit native magic from trezor-common --- configs/coins/bgold.json | 1 + configs/coins/digibyte.json | 1 + configs/coins/gamecredits.json | 1 + configs/coins/litecoin.json | 1 + configs/coins/litecoin_testnet.json | 2 ++ configs/coins/monacoin.json | 1 + configs/coins/myriad.json | 2 ++ configs/coins/qtum_testnet.json | 2 +- configs/coins/vertcoin.json | 1 + 9 files changed, 11 insertions(+), 1 deletion(-) diff --git a/configs/coins/bgold.json b/configs/coins/bgold.json index 02633bd7..e518024c 100644 --- a/configs/coins/bgold.json +++ b/configs/coins/bgold.json @@ -250,6 +250,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "slip44": 156, "additional_params": {} } diff --git a/configs/coins/digibyte.json b/configs/coins/digibyte.json index 5deb852d..4f879ee3 100644 --- a/configs/coins/digibyte.json +++ b/configs/coins/digibyte.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "slip44": 20, "additional_params": {} } diff --git a/configs/coins/gamecredits.json b/configs/coins/gamecredits.json index dfb5efb5..848f231d 100644 --- a/configs/coins/gamecredits.json +++ b/configs/coins/gamecredits.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 27106558, "xpub_magic_segwit_p2sh": 28471030, + "xpub_magic_segwit_native": 78792518, "slip44": 101, "additional_params": {} } diff --git a/configs/coins/litecoin.json b/configs/coins/litecoin.json index 600e8d34..09d637dc 100644 --- a/configs/coins/litecoin.json +++ b/configs/coins/litecoin.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 27108450, "xpub_magic_segwit_p2sh": 28471030, + "xpub_magic_segwit_native": 78792518, "slip44": 2, "additional_params": {} } diff --git a/configs/coins/litecoin_testnet.json b/configs/coins/litecoin_testnet.json index 622bf912..022f9217 100644 --- a/configs/coins/litecoin_testnet.json +++ b/configs/coins/litecoin_testnet.json @@ -56,6 +56,8 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 70617039, + "xpub_magic_segwit_p2sh": 71979618, + "xpub_magic_segwit_native": 73342198, "slip44": 1, "additional_params": {} } diff --git a/configs/coins/monacoin.json b/configs/coins/monacoin.json index db281c81..4cdc4e91 100644 --- a/configs/coins/monacoin.json +++ b/configs/coins/monacoin.json @@ -57,6 +57,7 @@ "block_addresses_to_keep": 300, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "slip44": 22, "additional_params": {} } diff --git a/configs/coins/myriad.json b/configs/coins/myriad.json index 1e831b0a..f821439b 100644 --- a/configs/coins/myriad.json +++ b/configs/coins/myriad.json @@ -56,6 +56,8 @@ "mempool_sub_workers": 2, "block_addresses_to_keep": 300, "xpub_magic": 76067358, + "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "slip44": 90, "additional_params": {} } diff --git a/configs/coins/qtum_testnet.json b/configs/coins/qtum_testnet.json index 525131b3..70291ffa 100644 --- a/configs/coins/qtum_testnet.json +++ b/configs/coins/qtum_testnet.json @@ -1,7 +1,7 @@ { "coin": { "name": "Qtum Testnet", - "shortcut": "TQTUM", + "shortcut": "tQTUM", "label": "Qtum Testnet", "alias": "qtum_testnet" }, diff --git a/configs/coins/vertcoin.json b/configs/coins/vertcoin.json index 0dfa7545..1ce7fa77 100644 --- a/configs/coins/vertcoin.json +++ b/configs/coins/vertcoin.json @@ -55,6 +55,7 @@ "block_addresses_to_keep": 1000, "xpub_magic": 76067358, "xpub_magic_segwit_p2sh": 77429938, + "xpub_magic_segwit_native": 78792518, "slip44": 28, "additional_params": {} } From e6863b042078a71ab81ba1c66f80f55af1899e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Wed, 27 Mar 2019 13:11:57 +0100 Subject: [PATCH 03/39] Bump Dash backend version to 0.13.2.0 --- configs/coins/dash.json | 8 ++++---- configs/coins/dash_testnet.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configs/coins/dash.json b/configs/coins/dash.json index 011b5f07..b0198330 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -22,10 +22,10 @@ "package_name": "backend-dash", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.13.1.0", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.1.0/dashcore-0.13.1.0-x86_64-linux-gnu.tar.gz", + "version": "0.13.2.0", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/dashcore-0.13.2.0-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.1.0/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" @@ -65,4 +65,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} \ No newline at end of file +} diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index ffbeb63c..0d6952ca 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-dash-testnet", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.13.1.0", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.1.0/dashcore-0.13.1.0-x86_64-linux-gnu.tar.gz", + "version": "0.13.2.0", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/dashcore-0.13.2.0-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.1.0/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" @@ -65,4 +65,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} \ No newline at end of file +} From 57ce8742087701bc9a30c5f8a1c4a4fcf245b486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Wed, 27 Mar 2019 13:16:15 +0100 Subject: [PATCH 04/39] Bump BCash backend version to 0.19.1 --- configs/coins/bcash.json | 8 ++++---- configs/coins/bcash_testnet.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 569280e5..84c22b34 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.19.0", - "binary_url": "https://download.bitcoinabc.org/0.19.0/linux/bitcoin-abc-0.19.0-x86_64-linux-gnu.tar.gz", + "version": "0.19.1", + "binary_url": "https://download.bitcoinabc.org/0.19.1/linux/bitcoin-abc-0.19.1-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "788001fd5ce8ca0bd61e8a92f10d22b7a49695c3651c60fad11358ba29309a1b", + "verification_source": "ffea0367af81fdc2f50e780938c6a1619aec1be125134777fff8a024b56923d1", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -63,4 +63,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} \ No newline at end of file +} diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index dfd8308a..646763b6 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash-testnet", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.19.0", - "binary_url": "https://download.bitcoinabc.org/0.19.0/linux/bitcoin-abc-0.19.0-x86_64-linux-gnu.tar.gz", + "version": "0.19.1", + "binary_url": "https://download.bitcoinabc.org/0.19.1/linux/bitcoin-abc-0.19.1-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "788001fd5ce8ca0bd61e8a92f10d22b7a49695c3651c60fad11358ba29309a1b", + "verification_source": "ffea0367af81fdc2f50e780938c6a1619aec1be125134777fff8a024b56923d1", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" @@ -63,4 +63,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} \ No newline at end of file +} From ee4ecc2bb0294bb0fbeefb24b89de1f596d5db82 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 27 Mar 2019 14:25:36 +0100 Subject: [PATCH 05/39] Fix bcash EstimateFee after interface change in backend v0.19.1 --- bchain/coins/bch/bcashrpc.go | 33 +++++++++++++++++++++++++++++++++ configs/coins/bcash.json | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index 36a955c1..a57154bb 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -5,6 +5,7 @@ import ( "blockbook/bchain/coins/btc" "encoding/hex" "encoding/json" + "math/big" "github.com/golang/glog" "github.com/juju/errors" @@ -162,3 +163,35 @@ func isErrBlockNotFound(err *bchain.RPCError) bool { return err.Message == "Block not found" || err.Message == "Block height out of range" } + +// EstimateFee returns fee estimation +func (b *BCashRPC) EstimateFee(blocks int) (big.Int, error) { + // from version BitcoinABC version 0.19.1 EstimateFee does not support parameter Blocks + if b.ChainConfig.CoinShortcut == "BCHSV" { + return b.BitcoinRPC.EstimateFee(blocks) + } + + glog.V(1).Info("rpc: estimatefee ", blocks) + + res := btc.ResEstimateFee{} + req := struct { + Method string `json:"method"` + }{ + Method: "estimatefee", + } + + err := b.Call(&req, &res) + + var r big.Int + if err != nil { + return r, err + } + if res.Error != nil { + return r, res.Error + } + r, err = b.Parser.AmountToBigInt(res.Result) + if err != nil { + return r, err + } + return r, nil +} diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 84c22b34..efe85e93 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -49,7 +49,7 @@ "additional_params": "", "block_chain": { "parse": true, - "subversion": "/Bitcoin ABC:0.17.1/", + "subversion": "/Bitcoin ABC:0.19.1/", "address_format": "cashaddr", "mempool_workers": 8, "mempool_sub_workers": 2, From 1cd4c207be381b68dbf85e5eb3b0fd3a526e903e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 28 Mar 2019 11:09:57 +0100 Subject: [PATCH 06/39] Update xpub derivation documentation --- docs/api.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 26a7bfc2..b028268d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -269,7 +269,13 @@ Response: #### Get xpub -Returns balances and transactions of an xpub, applicable only for Bitcoin-type coins. Xpub must be in BIP44, BIP49 or BIP84 format (the BIP number is detected by the prefix of xpub) with level 3 derivation path, i.e. *m/purpose'/coin_type'/account'/*. Blockbook completes the *change/address_index* part of the path when deriving addresses. The returned transactions are sorted by block height, newest blocks first. +Returns balances and transactions of an xpub, applicable only for Bitcoin-type coins. + +Blockbook supports BIP44, BIP49 and BIP84 derivation schemes. It expects xpub at level 3 derivation path, i.e. *m/purpose'/coin_type'/account'/*. Blockbook completes the *change/address_index* part of the path when deriving addresses. + +The BIP version is determined by the prefix of the xpub. The prefixes for each coin are defined by fields `xpub_magic`, `xpub_magic_segwit_p2sh`, `xpub_magic_segwit_native` in the [trezor-common](https://github.com/trezor/trezor-common/tree/master/defs/bitcoin) library. If the prefix is not recognized, Blockbook defaults to BIP44 derivation scheme. + +The returned transactions are sorted by block height, newest blocks first. ``` GET /api/v2/xpub/[?page=&pageSize=&from=&to=&details=&tokens=] From 72f4ad9101f3008644954e96168b23308b50aa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Thu, 28 Mar 2019 10:56:42 +0100 Subject: [PATCH 07/39] Bump BCash backend version to 0.19.2 --- configs/coins/bcash.json | 6 +++--- configs/coins/bcash_testnet.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index efe85e93..6c50aec0 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.19.1", - "binary_url": "https://download.bitcoinabc.org/0.19.1/linux/bitcoin-abc-0.19.1-x86_64-linux-gnu.tar.gz", + "version": "0.19.2", + "binary_url": "https://download.bitcoinabc.org/0.19.2/linux/bitcoin-abc-0.19.2-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "ffea0367af81fdc2f50e780938c6a1619aec1be125134777fff8a024b56923d1", + "verification_source": "615063ef2049f756c2cbb0b0823964a5eed5c5dd5420d10049a0b30563f631d2", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index 646763b6..1c6fee74 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash-testnet", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.19.1", - "binary_url": "https://download.bitcoinabc.org/0.19.1/linux/bitcoin-abc-0.19.1-x86_64-linux-gnu.tar.gz", + "version": "0.19.2", + "binary_url": "https://download.bitcoinabc.org/0.19.2/linux/bitcoin-abc-0.19.2-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "ffea0367af81fdc2f50e780938c6a1619aec1be125134777fff8a024b56923d1", + "verification_source": "615063ef2049f756c2cbb0b0823964a5eed5c5dd5420d10049a0b30563f631d2", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" From 9cd35b3616dc7117f635995ba033931205b32f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Thu, 28 Mar 2019 10:59:58 +0100 Subject: [PATCH 08/39] Bump ZCash backend version to 2.0.4 --- configs/coins/zcash.json | 8 ++++---- configs/coins/zcash_testnet.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configs/coins/zcash.json b/configs/coins/zcash.json index f4f3c802..4c635f56 100644 --- a/configs/coins/zcash.json +++ b/configs/coins/zcash.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "2.0.3", - "binary_url": "https://z.cash/downloads/zcash-2.0.3-linux64.tar.gz", + "version": "2.0.4", + "binary_url": "https://z.cash/downloads/zcash-2.0.4-linux64.tar.gz", "verification_type": "sha256", - "verification_source": "529900fa311509274cc3912ae886a3b8e193ae8a91801f810c3ca58f44bd1730", + "verification_source": "a825060f425128260b603e3524c472f4ba28ba17090bece4ac38c84b1bcf5940", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -64,4 +64,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} \ No newline at end of file +} diff --git a/configs/coins/zcash_testnet.json b/configs/coins/zcash_testnet.json index 8dd220a2..3fc370ff 100644 --- a/configs/coins/zcash_testnet.json +++ b/configs/coins/zcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-zcash-testnet", "package_revision": "satoshilabs-1", "system_user": "zcash", - "version": "2.0.3", - "binary_url": "https://z.cash/downloads/zcash-2.0.3-linux64.tar.gz", + "version": "2.0.4", + "binary_url": "https://z.cash/downloads/zcash-2.0.4-linux64.tar.gz", "verification_type": "sha256", - "verification_source": "529900fa311509274cc3912ae886a3b8e193ae8a91801f810c3ca58f44bd1730", + "verification_source": "a825060f425128260b603e3524c472f4ba28ba17090bece4ac38c84b1bcf5940", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/zcashd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", @@ -64,4 +64,4 @@ "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} \ No newline at end of file +} From fa4a11c3a7ebfd9e66bcb50cc74896bb3c5a47f3 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 28 Mar 2019 11:24:20 +0100 Subject: [PATCH 09/39] Fix bcash EstimateSmartFee functionality --- bchain/coins/bch/bcashrpc.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bchain/coins/bch/bcashrpc.go b/bchain/coins/bch/bcashrpc.go index a57154bb..79500773 100644 --- a/bchain/coins/bch/bcashrpc.go +++ b/bchain/coins/bch/bcashrpc.go @@ -195,3 +195,9 @@ func (b *BCashRPC) EstimateFee(blocks int) (big.Int, error) { } return r, nil } + +// EstimateSmartFee returns fee estimation +func (b *BCashRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + // EstimateSmartFee is not supported by bcash + return b.EstimateFee(blocks) +} From 954619efe95b512aa20d92c5e0d0dbb77ddef88e Mon Sep 17 00:00:00 2001 From: kiss1987f4 <156755027@qq.com> Date: Thu, 28 Mar 2019 21:31:29 +0800 Subject: [PATCH 10/39] Add NULS supported. (#135) Add NULS supported. --- bchain/coins/blockchain.go | 2 + bchain/coins/nuls/nulsparser.go | 151 +++++++ bchain/coins/nuls/nulsparser_test.go | 343 +++++++++++++++ bchain/coins/nuls/nulsrpc.go | 635 +++++++++++++++++++++++++++ configs/coins/nuls.json | 67 +++ tests/rpc/testdata/nuls.json | 174 ++++++++ tests/sync/testdata/nuls.json | 546 +++++++++++++++++++++++ tests/tests.json | 4 + 8 files changed, 1922 insertions(+) create mode 100644 bchain/coins/nuls/nulsparser.go create mode 100644 bchain/coins/nuls/nulsparser_test.go create mode 100644 bchain/coins/nuls/nulsrpc.go create mode 100644 configs/coins/nuls.json create mode 100644 tests/rpc/testdata/nuls.json create mode 100644 tests/sync/testdata/nuls.json diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 34871e75..6d1cdcb6 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -20,6 +20,7 @@ import ( "blockbook/bchain/coins/monacoin" "blockbook/bchain/coins/myriad" "blockbook/bchain/coins/namecoin" + "blockbook/bchain/coins/nuls" "blockbook/bchain/coins/pivx" "blockbook/bchain/coins/qtum" "blockbook/bchain/coins/vertcoin" @@ -79,6 +80,7 @@ func init() { BlockChainFactories["Bellcoin"] = bellcoin.NewBellcoinRPC BlockChainFactories["Qtum"] = qtum.NewQtumRPC BlockChainFactories["Qtum Testnet"] = qtum.NewQtumRPC + BlockChainFactories["NULS"] = nuls.NewNulsRPC } // GetCoinNameFromConfig gets coin name and coin shortcut from config file diff --git a/bchain/coins/nuls/nulsparser.go b/bchain/coins/nuls/nulsparser.go new file mode 100644 index 00000000..0b6275c1 --- /dev/null +++ b/bchain/coins/nuls/nulsparser.go @@ -0,0 +1,151 @@ +package nuls + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "bytes" + "encoding/binary" + "encoding/json" + "github.com/bsm/go-vlq" + "github.com/martinboehm/btcutil/base58" + + "github.com/martinboehm/btcd/wire" + "github.com/martinboehm/btcutil/chaincfg" +) + +const ( + MainnetMagic wire.BitcoinNet = 0xbd6b0cbf + TestnetMagic wire.BitcoinNet = 0xffcae2ce + RegtestMagic wire.BitcoinNet = 0xdcb7c1fc +) + +var ( + MainNetParams chaincfg.Params + TestNetParams chaincfg.Params + RegtestParams chaincfg.Params +) + +func init() { + MainNetParams = chaincfg.MainNetParams + MainNetParams.Net = MainnetMagic + + // Address encoding magics + MainNetParams.PubKeyHashAddrID = []byte{38} // base58 prefix: G + MainNetParams.ScriptHashAddrID = []byte{10} // base58 prefix: W + + TestNetParams = chaincfg.TestNet3Params + TestNetParams.Net = TestnetMagic + + // Address encoding magics + TestNetParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + TestNetParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 + + RegtestParams = chaincfg.RegressionNetParams + RegtestParams.Net = RegtestMagic + + // Address encoding magics + RegtestParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + RegtestParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 +} + +// NulsParser handle +type NulsParser struct { + *btc.BitcoinParser +} + +// NewNulsParser returns new NulsParser instance +func NewNulsParser(params *chaincfg.Params, c *btc.Configuration) *NulsParser { + return &NulsParser{BitcoinParser: btc.NewBitcoinParser(params, c)} +} + +// GetChainParams contains network parameters for the main Gincoin network, +// the regression test Gincoin network, the test Gincoin network and +// the simulation test Gincoin network, in this order +func GetChainParams(chain string) *chaincfg.Params { + if !chaincfg.IsRegistered(&MainNetParams) { + err := chaincfg.Register(&MainNetParams) + if err == nil { + err = chaincfg.Register(&TestNetParams) + } + if err == nil { + err = chaincfg.Register(&RegtestParams) + } + if err != nil { + panic(err) + } + } + switch chain { + case "test": + return &TestNetParams + case "regtest": + return &RegtestParams + default: + return &MainNetParams + } +} + +// PackedTxidLen returns length in bytes of packed txid +func (p *NulsParser) PackedTxidLen() int { + return 34 +} + +// GetAddrDescFromAddress returns internal address representation (descriptor) of given address +func (p *NulsParser) GetAddrDescFromAddress(address string) (bchain.AddressDescriptor, error) { + addressByte := base58.Decode(address) + return bchain.AddressDescriptor(addressByte), nil +} + +// GetAddrDescFromVout returns internal address representation (descriptor) of given transaction output +func (p *NulsParser) GetAddrDescFromVout(output *bchain.Vout) (bchain.AddressDescriptor, error) { + addressStr := output.ScriptPubKey.Hex + addressByte := base58.Decode(addressStr) + return bchain.AddressDescriptor(addressByte), nil +} + +// GetAddressesFromAddrDesc returns addresses for given address descriptor with flag if the addresses are searchable +func (p *NulsParser) GetAddressesFromAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, bool, error) { + var addrs []string + + if addrDesc != nil { + addrs = append(addrs, base58.Encode(addrDesc)) + } + + return addrs, true, nil +} + +// PackTx packs transaction to byte array +func (p *NulsParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { + txBytes, error := json.Marshal(tx) + if error != nil { + return nil, error + } + + buf := make([]byte, 4+vlq.MaxLen64) + binary.BigEndian.PutUint32(buf[0:4], height) + vlq.PutInt(buf[4:4+vlq.MaxLen64], blockTime) + resByes := bytes.Join([][]byte{buf, txBytes}, []byte("")) + return resByes, nil +} + +// UnpackTx unpacks transaction from byte array +func (p *NulsParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { + height := binary.BigEndian.Uint32(buf) + bt, _ := vlq.Int(buf[4 : 4+vlq.MaxLen64]) + tx, err := p.ParseTx(buf[4+vlq.MaxLen64:]) + if err != nil { + return nil, 0, err + } + tx.Blocktime = bt + + return tx, height, nil +} + +func (p *NulsParser) ParseTx(b []byte) (*bchain.Tx, error) { + tx := bchain.Tx{} + err := json.Unmarshal(b, &tx) + + if err != nil { + return nil, err + } + return &tx, err +} diff --git a/bchain/coins/nuls/nulsparser_test.go b/bchain/coins/nuls/nulsparser_test.go new file mode 100644 index 00000000..614000cd --- /dev/null +++ b/bchain/coins/nuls/nulsparser_test.go @@ -0,0 +1,343 @@ +package nuls + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "encoding/hex" + "encoding/json" + "math/big" + "reflect" + "testing" +) + +var ( + testTx1, testTx2 bchain.Tx + + testTxPacked1 = "0001e240daadfbe7931e000000007b22686578223a22222c2274786964223a223030323036626231323431303861356664393865383238623138316666303162393237633063366234373764343531326266656638346366353266306663326136613161222c2276657273696f6e223a312c226c6f636b74696d65223a302c2276696e223a5b7b22636f696e62617365223a22222c2274786964223a223030323035343537616230373033623164313034363264343334373034386330626233353634653430663537616531663632353136393539343364653161633831306130222c22766f7574223a302c22736372697074536967223a7b22686578223a22227d2c2273657175656e6365223a302c22616464726573736573223a5b224e736535334d77524c424a31575555365365644d485141466643507442377734225d7d5d2c22766f7574223a5b7b2256616c7565536174223a3339393939393030303030302c2276616c7565223a302c226e223a302c227363726970745075624b6579223a7b22686578223a224e7365347a705a4873557555376835796d7632387063476277486a75336a6f56222c22616464726573736573223a5b224e7365347a705a4873557555376835796d7632387063476277486a75336a6f56225d7d7d5d2c22626c6f636b74696d65223a313535323335373834343137357d" + testTxPacked2 = "0007c91adaadfbb89946000000007b22686578223a22222c2274786964223a223030323037386139386633383163373134613036386436303565346265316565323139353438353736313165303938616262636333663530633536383066386164326535222c2276657273696f6e223a312c226c6f636b74696d65223a302c2276696e223a5b7b22636f696e62617365223a22222c2274786964223a223030323037613430646334623661633430376434396133633333616137353462623466303565343565353763323438313162313437653762663363616630363361383233222c22766f7574223a312c22736372697074536967223a7b22686578223a22227d2c2273657175656e6365223a302c22616464726573736573223a5b224e73653131397a326f53444a596b466b786d775944695974506642654e6b7169225d7d5d2c22766f7574223a5b7b2256616c7565536174223a3430303030303030303030302c2276616c7565223a302c226e223a302c227363726970745075624b6579223a7b22686578223a224e736534696b6a45383867324267734e7773737754646b53776953724b6a6a53222c22616464726573736573223a5b224e736534696b6a45383867324267734e7773737754646b53776953724b6a6a53225d7d7d2c7b2256616c7565536174223a373238363536353537303030302c2276616c7565223a302c226e223a312c227363726970745075624b6579223a7b22686578223a224e73653131397a326f53444a596b466b786d775944695974506642654e6b7169222c22616464726573736573223a5b224e73653131397a326f53444a596b466b786d775944695974506642654e6b7169225d7d7d5d2c22626c6f636b74696d65223a313535323335373435393535357d" +) + +func init() { + testTx1 = bchain.Tx{ + Hex: "", + Blocktime: 1552357844175, + Txid: "00206bb124108a5fd98e828b181ff01b927c0c6b477d4512bfef84cf52f0fc2a6a1a", + LockTime: 0, + Version: 1, + Vin: []bchain.Vin{ + { + Txid: "00205457ab0703b1d10462d4347048c0bb3564e40f57ae1f6251695943de1ac810a0", + Vout: 0, + Sequence: 0, + Addresses: []string{ + "Nse53MwRLBJ1WUU6SedMHQAFfCPtB7w4", + }, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(399999000000), + N: 0, + JsonValue: json.Number("0"), + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "Nse4zpZHsUuU7h5ymv28pcGbwHju3joV", + Addresses: []string{ + "Nse4zpZHsUuU7h5ymv28pcGbwHju3joV", + }, + }, + }, + }, + //CoinSpecificData: []string{}, + } + + testTx2 = bchain.Tx{ + Hex: "", + Blocktime: 1552357459555, + Txid: "002078a98f381c714a068d605e4be1ee21954857611e098abbcc3f50c5680f8ad2e5", + LockTime: 0, + Version: 1, + Vin: []bchain.Vin{ + { + Txid: "00207a40dc4b6ac407d49a3c33aa754bb4f05e45e57c24811b147e7bf3caf063a823", + Vout: 1, + Sequence: 0, + Addresses: []string{ + "Nse119z2oSDJYkFkxmwYDiYtPfBeNkqi", + }, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(400000000000), + N: 0, + JsonValue: json.Number("0"), + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "Nse4ikjE88g2BgsNwsswTdkSwiSrKjjS", + Addresses: []string{ + "Nse4ikjE88g2BgsNwsswTdkSwiSrKjjS", + }, + }, + }, + { + ValueSat: *big.NewInt(7286565570000), + N: 1, + JsonValue: json.Number("0"), + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "Nse119z2oSDJYkFkxmwYDiYtPfBeNkqi", + Addresses: []string{ + "Nse119z2oSDJYkFkxmwYDiYtPfBeNkqi", + }, + }, + }, + }, + //CoinSpecificData: []string{}, + } +} + +func TestGetAddrDescFromAddress(t *testing.T) { + type args struct { + address string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "P2PKH", + args: args{address: "Nse4j39uEMuxx5j577h3K2MDLAQ64JZN"}, + want: "042301ac78cd3eb193287e2c59e4cbd765c5c47d432c2fa1", + wantErr: false, + }, + { + name: "P2PKH", + args: args{address: "Nse2e7U7nmGT8UHsvQ7JfksLtWwoLwrd"}, + want: "0423018a90e66a64318f6af6d673487a6560f5686fd26a2e", + wantErr: false, + }, + { + name: "P2PKH", + args: args{address: "NsdvMEP57nzxmBa5z18rx9sQsLgUfNtw"}, + want: "04230124422cfe426573e476fd45d7c2a43a75a0b6b8c478", + wantErr: false, + }, + } + parser := NewNulsParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parser.GetAddrDescFromAddress(tt.args.address) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) + } + }) + } +} + +func TestGetAddrDescFromVout(t *testing.T) { + type args struct { + vout bchain.Vout + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "P2PK", + args: args{vout: bchain.Vout{ScriptPubKey: bchain.ScriptPubKey{Hex: "Nse4j39uEMuxx5j577h3K2MDLAQ64JZN"}}}, + want: "042301ac78cd3eb193287e2c59e4cbd765c5c47d432c2fa1", + wantErr: false, + }, + { + name: "P2PK", + args: args{vout: bchain.Vout{ScriptPubKey: bchain.ScriptPubKey{Hex: "Nse2e7U7nmGT8UHsvQ7JfksLtWwoLwrd"}}}, + want: "0423018a90e66a64318f6af6d673487a6560f5686fd26a2e", + wantErr: false, + }, + { + name: "P2PK", + args: args{vout: bchain.Vout{ScriptPubKey: bchain.ScriptPubKey{Hex: "NsdvMEP57nzxmBa5z18rx9sQsLgUfNtw"}}}, + want: "04230124422cfe426573e476fd45d7c2a43a75a0b6b8c478", + wantErr: false, + }, + } + parser := NewNulsParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parser.GetAddrDescFromVout(&tt.args.vout) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddrDescFromVout() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("GetAddrDescFromVout() = %v, want %v", h, tt.want) + } + }) + } +} + +func TestGetAddressesFromAddrDesc(t *testing.T) { + type args struct { + script string + } + tests := []struct { + name string + args args + want []string + want2 bool + wantErr bool + }{ + { + name: "P2PKH", + args: args{script: "042301ac78cd3eb193287e2c59e4cbd765c5c47d432c2fa1"}, + want: []string{"Nse4j39uEMuxx5j577h3K2MDLAQ64JZN"}, + want2: true, + wantErr: false, + }, + { + name: "P2PKH", + args: args{script: "0423018a90e66a64318f6af6d673487a6560f5686fd26a2e"}, + want: []string{"Nse2e7U7nmGT8UHsvQ7JfksLtWwoLwrd"}, + want2: true, + wantErr: false, + }, + { + name: "P2PKH", + args: args{script: "04230124422cfe426573e476fd45d7c2a43a75a0b6b8c478"}, + want: []string{"NsdvMEP57nzxmBa5z18rx9sQsLgUfNtw"}, + want2: true, + wantErr: false, + }, + } + + parser := NewNulsParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.script) + got, got2, err := parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) + } + }) + } +} + +func TestPackTx(t *testing.T) { + type args struct { + tx bchain.Tx + height uint32 + blockTime int64 + parser *NulsParser + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test-1", + args: args{ + tx: testTx1, + height: 123456, + blockTime: 1552357844175, + parser: NewNulsParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: testTxPacked1, + wantErr: false, + }, + { + name: "test-2", + args: args{ + tx: testTx2, + height: 510234, + blockTime: 1552357459555, + parser: NewNulsParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: testTxPacked2, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.parser.PackTx(&tt.args.tx, tt.args.height, tt.args.blockTime) + if (err != nil) != tt.wantErr { + t.Errorf("packTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("packTx() = %v, want %v", h, tt.want) + } + }) + } +} + +func TestUnpackTx(t *testing.T) { + type args struct { + packedTx string + parser *NulsParser + } + tests := []struct { + name string + args args + want *bchain.Tx + want1 uint32 + wantErr bool + }{ + { + name: "test-1", + args: args{ + packedTx: testTxPacked1, + parser: NewNulsParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: &testTx1, + want1: 123456, + wantErr: false, + }, + { + name: "test-2", + args: args{ + packedTx: testTxPacked2, + parser: NewNulsParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: &testTx2, + want1: 510234, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.packedTx) + got, got1, err := tt.args.parser.UnpackTx(b) + if (err != nil) != tt.wantErr { + t.Errorf("unpackTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unpackTx() got = %+v, want %+v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("unpackTx() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/bchain/coins/nuls/nulsrpc.go b/bchain/coins/nuls/nulsrpc.go new file mode 100644 index 00000000..3bd7b24e --- /dev/null +++ b/bchain/coins/nuls/nulsrpc.go @@ -0,0 +1,635 @@ +package nuls + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "bytes" + "encoding/base64" + "encoding/json" + "github.com/gobuffalo/packr/v2/file/resolver/encoding/hex" + "github.com/juju/errors" + "io" + "io/ioutil" + "math/big" + "net" + "net/http" + "runtime/debug" + "strconv" + "time" + + "github.com/golang/glog" +) + +// NulsRPC is an interface to JSON-RPC bitcoind service. +type NulsRPC struct { + *btc.BitcoinRPC + client http.Client + rpcURL string + user string + password string +} + +// NewNulsRPC returns new NulsRPC instance. +func NewNulsRPC(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) { + b, err := btc.NewBitcoinRPC(config, pushHandler) + if err != nil { + return nil, err + } + + var c btc.Configuration + err = json.Unmarshal(config, &c) + if err != nil { + return nil, errors.Annotatef(err, "Invalid configuration file") + } + + transport := &http.Transport{ + Dial: (&net.Dialer{KeepAlive: 600 * time.Second}).Dial, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, // necessary to not to deplete ports + } + + s := &NulsRPC{ + BitcoinRPC: b.(*btc.BitcoinRPC), + client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport}, + rpcURL: c.RPCURL, + user: c.RPCUser, + password: c.RPCPass, + } + s.BitcoinRPC.RPCMarshaler = btc.JSONMarshalerV1{} + s.BitcoinRPC.ChainConfig.SupportsEstimateSmartFee = false + + return s, nil +} + +// Initialize initializes GincoinRPC instance. +func (n *NulsRPC) Initialize() error { + chainName := "" + params := GetChainParams(chainName) + + // always create parser + n.BitcoinRPC.Parser = NewNulsParser(params, n.BitcoinRPC.ChainConfig) + + // parameters for getInfo request + if params.Net == MainnetMagic { + n.BitcoinRPC.Testnet = false + n.BitcoinRPC.Network = "livenet" + } else { + n.BitcoinRPC.Testnet = true + n.BitcoinRPC.Network = "testnet" + } + + glog.Info("rpc: block chain ", params.Name) + + return nil +} + +type CmdGetNetworkInfo struct { + Success bool `json:"success"` + Data struct { + LocalBestHeight int64 `json:"localBestHeight"` + NetBestHeight int `json:"netBestHeight"` + TimeOffset string `json:"timeOffset"` + InCount int8 `json:"inCount"` + OutCount int8 `json:"outCount"` + } `json:"data"` +} + +type CmdGetVersionInfo struct { + Success bool `json:"success"` + Data struct { + MyVersion string `json:"myVersion"` + NewestVersion string `json:"newestVersion"` + NetworkVersion int `json:"networkVersion"` + Infromation string `json:"infromation"` + } `json:"data"` +} + +type CmdGetBestBlockHash struct { + Success bool `json:"success"` + Data struct { + Value string `json:"value"` + } `json:"data"` +} + +type CmdGetBestBlockHeight struct { + Success bool `json:"success"` + Data struct { + Value uint32 `json:"value"` + } `json:"data"` +} + +type CmdTxBroadcast struct { + Success bool `json:"success"` + Data struct { + Value string `json:"value"` + } `json:"data"` +} + +type CmdGetBlockHeader struct { + Success bool `json:"success"` + Data struct { + Hash string `json:"hash"` + PreHash string `json:"preHash"` + MerkleHash string `json:"merkleHash"` + StateRoot string `json:"stateRoot"` + Time int64 `json:"time"` + Height int64 `json:"height"` + TxCount int `json:"txCount"` + PackingAddress string `json:"packingAddress"` + ConfirmCount int `json:"confirmCount"` + ScriptSig string `json:"scriptSig"` + Size int `json:"size"` + Reward float64 `json:"reward"` + Fee float64 `json:"fee"` + } `json:"data"` +} + +type CmdGetBlock struct { + Success bool `json:"success"` + Data struct { + Hash string `json:"hash"` + PreHash string `json:"preHash"` + MerkleHash string `json:"merkleHash"` + StateRoot string `json:"stateRoot"` + Time int64 `json:"time"` + Height int64 `json:"height"` + TxCount int `json:"txCount"` + PackingAddress string `json:"packingAddress"` + ConfirmCount int `json:"confirmCount"` + ScriptSig string `json:"scriptSig"` + Size int `json:"size"` + Reward float64 `json:"reward"` + Fee float64 `json:"fee"` + + TxList []Tx `json:"txList"` + } `json:"data"` +} + +type CmdGetTx struct { + Success bool `json:"success"` + Tx Tx `json:"data"` +} + +type Tx struct { + Hash string `json:"hash"` + Type int `json:"type"` + Time int64 `json:"time"` + BlockHeight int64 `json:"blockHeight"` + Fee float64 `json:"fee"` + Value float64 `json:"value"` + Remark string `json:"remark"` + ScriptSig string `json:"scriptSig"` + Status int `json:"status"` + ConfirmCount int `json:"confirmCount"` + Size int `json:"size"` + Inputs []struct { + FromHash string `json:"fromHash"` + FromIndex uint32 `json:"fromIndex"` + Address string `json:"address"` + Value float64 `json:"value"` + } `json:"inputs"` + Outputs []struct { + Address string `json:"address"` + Value int64 `json:"value"` + LockTime int64 `json:"lockTime"` + } `json:"outputs"` +} + +type CmdGetTxBytes struct { + Success bool `json:"success"` + Data struct { + Value string `json:"value"` + } `json:"data"` +} + +func (n *NulsRPC) GetChainInfo() (*bchain.ChainInfo, error) { + networkInfo := CmdGetNetworkInfo{} + error := n.Call("/api/network/info", &networkInfo) + if error != nil { + return nil, error + } + + versionInfo := CmdGetVersionInfo{} + error = n.Call("/api/client/version", &versionInfo) + if error != nil { + return nil, error + } + + chainInfo := &bchain.ChainInfo{ + Chain: "nuls", + Blocks: networkInfo.Data.NetBestHeight, + Headers: networkInfo.Data.NetBestHeight, + Bestblockhash: "", + Difficulty: networkInfo.Data.TimeOffset, + SizeOnDisk: networkInfo.Data.LocalBestHeight, + Version: versionInfo.Data.MyVersion, + Subversion: versionInfo.Data.NewestVersion, + ProtocolVersion: strconv.Itoa(versionInfo.Data.NetworkVersion), + Timeoffset: 0, + Warnings: versionInfo.Data.Infromation, + } + return chainInfo, nil +} + +func (n *NulsRPC) GetBestBlockHash() (string, error) { + bestBlockHash := CmdGetBestBlockHash{} + error := n.Call("/api/block/newest/hash", &bestBlockHash) + if error != nil { + return "", error + } + return bestBlockHash.Data.Value, nil +} + +func (n *NulsRPC) GetBestBlockHeight() (uint32, error) { + bestBlockHeight := CmdGetBestBlockHeight{} + error := n.Call("/api/block/newest/height", &bestBlockHeight) + if error != nil { + return 0, error + } + return bestBlockHeight.Data.Value, nil +} + +func (n *NulsRPC) GetBlockHash(height uint32) (string, error) { + blockHeader := CmdGetBlockHeader{} + error := n.Call("/api/block/header/height/"+strconv.Itoa(int(height)), &blockHeader) + if error != nil { + return "", error + } + + if !blockHeader.Success { + return "", bchain.ErrBlockNotFound + } + + return blockHeader.Data.Hash, nil +} + +func (n *NulsRPC) GetBlockHeader(hash string) (*bchain.BlockHeader, error) { + uri := "/api/block/header/hash/" + hash + return n.getBlobkHeader(uri) +} + +func (n *NulsRPC) GetBlockHeaderByHeight(height uint32) (*bchain.BlockHeader, error) { + uri := "/api/block/header/height/" + strconv.Itoa(int(height)) + return n.getBlobkHeader(uri) +} + +func (n *NulsRPC) getBlobkHeader(uri string) (*bchain.BlockHeader, error) { + blockHeader := CmdGetBlockHeader{} + error := n.Call(uri, &blockHeader) + if error != nil { + return nil, error + } + + if !blockHeader.Success { + return nil, bchain.ErrBlockNotFound + } + + nexHash, _ := n.GetBlockHash(uint32(blockHeader.Data.Height + 1)) + + header := &bchain.BlockHeader{ + Hash: blockHeader.Data.Hash, + Prev: blockHeader.Data.PreHash, + Next: nexHash, + Height: uint32(blockHeader.Data.Height), + Confirmations: blockHeader.Data.ConfirmCount, + Size: blockHeader.Data.Size, + Time: blockHeader.Data.Time / 1000, + } + return header, nil +} + +func (n *NulsRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { + + url := "/api/block/hash/" + hash + + if hash == "" { + url = "/api/block/height/" + strconv.Itoa(int(height)) + } + + getBlock := CmdGetBlock{} + error := n.Call(url, &getBlock) + if error != nil { + return nil, error + } + + if !getBlock.Success { + return nil, bchain.ErrBlockNotFound + } + + nexHash, _ := n.GetBlockHash(uint32(getBlock.Data.Height + 1)) + + header := bchain.BlockHeader{ + Hash: getBlock.Data.Hash, + Prev: getBlock.Data.PreHash, + Next: nexHash, + Height: uint32(getBlock.Data.Height), + Confirmations: getBlock.Data.ConfirmCount, + Size: getBlock.Data.Size, + Time: getBlock.Data.Time / 1000, + } + + var txs []bchain.Tx + + for _, rawTx := range getBlock.Data.TxList { + tx, err := converTx(rawTx) + if err != nil { + return nil, err + } + tx.Blocktime = header.Time + txs = append(txs, *tx) + } + + block := &bchain.Block{ + BlockHeader: header, + Txs: txs, + } + return block, nil +} + +func (n *NulsRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { + if hash == "" { + return nil, bchain.ErrBlockNotFound + } + + getBlock := CmdGetBlock{} + error := n.Call("/api/block/hash/"+hash, &getBlock) + if error != nil { + return nil, error + } + + if !getBlock.Success { + return nil, bchain.ErrBlockNotFound + } + + nexHash, _ := n.GetBlockHash(uint32(getBlock.Data.Height + 1)) + + header := bchain.BlockHeader{ + Hash: getBlock.Data.Hash, + Prev: getBlock.Data.PreHash, + Next: nexHash, + Height: uint32(getBlock.Data.Height), + Confirmations: getBlock.Data.ConfirmCount, + Size: getBlock.Data.Size, + Time: getBlock.Data.Time / 1000, + } + + var txIds []string + + for _, rawTx := range getBlock.Data.TxList { + txIds = append(txIds, rawTx.Hash) + } + + blockInfo := &bchain.BlockInfo{ + BlockHeader: header, + MerkleRoot: getBlock.Data.MerkleHash, + //Version: getBlock.Data.StateRoot, + Txids: txIds, + } + return blockInfo, nil +} + +func (n *NulsRPC) GetMempool() ([]string, error) { + return nil, nil +} + +func (n *NulsRPC) GetTransaction(txid string) (*bchain.Tx, error) { + if txid == "" { + return nil, bchain.ErrTxidMissing + } + + getTx := CmdGetTx{} + error := n.Call("/api/tx/hash/"+txid, &getTx) + if error != nil { + return nil, error + } + + if !getTx.Success { + return nil, bchain.ErrTxNotFound + } + + tx, err := converTx(getTx.Tx) + if err != nil { + return nil, err + } + + blockHeaderHeight := getTx.Tx.BlockHeight + blockHeader, e := n.GetBlockHeaderByHeight(uint32(blockHeaderHeight)) + if blockHeader != nil { + tx.Blocktime = blockHeader.Time + } + + hexBytys, e := n.GetTransactionSpecific(tx) + if e == nil { + var hex string + json.Unmarshal(hexBytys, &hex) + tx.Hex = hex + tx.CoinSpecificData = hex + } + return tx, nil +} + +func (n *NulsRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { + return nil, nil +} + +func (n *NulsRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, error) { + if tx == nil { + return nil, bchain.ErrTxNotFound + } + + if csd, ok := tx.CoinSpecificData.(json.RawMessage); ok { + return csd, nil + } + + getTxBytes := CmdGetTxBytes{} + error := n.Call("/api/tx/bytes?hash="+tx.Txid, &getTxBytes) + if error != nil { + return nil, error + } + + if !getTxBytes.Success { + return nil, bchain.ErrTxNotFound + } + + txBytes, byErr := base64.StdEncoding.DecodeString(getTxBytes.Data.Value) + if byErr != nil { + return nil, byErr + } + hexBytes := make([]byte, len(txBytes)*2) + hex.Encode(hexBytes, txBytes) + + m, err := json.Marshal(string(hexBytes)) + return json.RawMessage(m), err +} + +func (n *NulsRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + return n.EstimateFee(blocks) +} + +func (n *NulsRPC) EstimateFee(blocks int) (big.Int, error) { + return *big.NewInt(100000), nil +} + +func (n *NulsRPC) SendRawTransaction(tx string) (string, error) { + broadcast := CmdTxBroadcast{} + req := struct { + TxHex string `json:"txHex"` + }{ + TxHex: tx, + } + + error := n.Post("/api/accountledger/transaction/broadcast", req, &broadcast) + if error != nil { + return "", error + } + + if !broadcast.Success { + return "", bchain.ErrTxidMissing + } + + return broadcast.Data.Value, nil +} + +func (n *NulsRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]bchain.Outpoint, error) { + return nil, nil +} + +func (n *NulsRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { + return 0, nil +} + +// Call calls Backend RPC interface, using RPCMarshaler interface to marshall the request +func (b *NulsRPC) Call(uri string, res interface{}) error { + url := b.rpcURL + uri + httpReq, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + httpReq.SetBasicAuth(b.user, b.password) + httpRes, err := b.client.Do(httpReq) + // in some cases the httpRes can contain data even if it returns error + // see http://devs.cloudimmunity.com/gotchas-and-common-mistakes-in-go-golang/ + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return err + } + // if server returns HTTP error code it might not return json with response + // handle both cases + if httpRes.StatusCode != 200 { + err = safeDecodeResponse(httpRes.Body, &res) + if err != nil { + return errors.Errorf("%v %v", httpRes.Status, err) + } + return nil + } + return safeDecodeResponse(httpRes.Body, &res) +} + +func (b *NulsRPC) Post(uri string, req interface{}, res interface{}) error { + url := b.rpcURL + uri + httpData, err := b.RPCMarshaler.Marshal(req) + if err != nil { + return err + } + httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(httpData)) + if err != nil { + return err + } + httpReq.SetBasicAuth(b.user, b.password) + httpReq.Header.Set("Content-Type", "application/json") + httpRes, err := b.client.Do(httpReq) + // in some cases the httpRes can contain data even if it returns error + // see http://devs.cloudimmunity.com/gotchas-and-common-mistakes-in-go-golang/ + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return err + } + // if server returns HTTP error code it might not return json with response + // handle both cases + if httpRes.StatusCode != 200 { + err = safeDecodeResponse(httpRes.Body, &res) + if err != nil { + return errors.Errorf("%v %v", httpRes.Status, err) + } + return nil + } + return safeDecodeResponse(httpRes.Body, &res) +} + +func safeDecodeResponse(body io.ReadCloser, res *interface{}) (err error) { + var data []byte + defer func() { + if r := recover(); r != nil { + glog.Error("unmarshal json recovered from panic: ", r, "; data: ", string(data)) + debug.PrintStack() + if len(data) > 0 && len(data) < 2048 { + err = errors.Errorf("Error: %v", string(data)) + } else { + err = errors.New("Internal error") + } + } + }() + data, err = ioutil.ReadAll(body) + if err != nil { + return err + } + + //fmt.Println(string(data)) + error := json.Unmarshal(data, res) + return error +} + +func converTx(rawTx Tx) (*bchain.Tx, error) { + var lockTime int64 = 0 + var vins = make([]bchain.Vin, 0) + var vouts []bchain.Vout + + for _, input := range rawTx.Inputs { + vin := bchain.Vin{ + Coinbase: "", + Txid: input.FromHash, + Vout: input.FromIndex, + ScriptSig: bchain.ScriptSig{}, + Sequence: 0, + Addresses: []string{input.Address}, + } + vins = append(vins, vin) + } + + for index, output := range rawTx.Outputs { + vout := bchain.Vout{ + ValueSat: *big.NewInt(output.Value), + //JsonValue: "", + //LockTime: output.LockTime, + N: uint32(index), + ScriptPubKey: bchain.ScriptPubKey{ + Hex: output.Address, + Addresses: []string{ + output.Address, + }, + }, + } + vouts = append(vouts, vout) + + if lockTime < output.LockTime { + lockTime = output.LockTime + } + } + + tx := &bchain.Tx{ + Hex: "", + Txid: rawTx.Hash, + Version: 0, + LockTime: uint32(lockTime), + Vin: vins, + Vout: vouts, + Confirmations: uint32(rawTx.ConfirmCount), + Time: rawTx.Time / 1000, + } + + return tx, nil +} diff --git a/configs/coins/nuls.json b/configs/coins/nuls.json new file mode 100644 index 00000000..29ce6308 --- /dev/null +++ b/configs/coins/nuls.json @@ -0,0 +1,67 @@ +{ + "coin": { + "name": "NULS", + "shortcut": "NULS", + "label": "NULS", + "alias": "nuls" + }, + "ports": { + "backend_rpc": 8053, + "backend_message_queue": 38353, + "blockbook_internal": 9053, + "blockbook_public": 9153 + }, + "ipc": { + "rpc_url_template": "http://127.0.0.1:{{.Ports.BackendRPC}}", + "rpc_user": "rpc", + "rpc_pass": "rpc", + "rpc_timeout": 25, + "message_queue_binding_template": "tcp://127.0.0.1:{{.Ports.BackendMessageQueue}}" + }, + "backend": { + "package_name": "backend-nuls", + "package_revision": "1.0-1", + "system_user": "nuls", + "version": "1.2.1", + "binary_url": "http://47.52.78.92/nuls-wallet/NULS-Wallet-linux64-1.2.1.tar.gz", + "verification_type": "sha256", + "verification_source": "8732e94415e9b6d51e9c7e1921002b4e43d78e2b12f2de636af53a562b4f4381", + "extract_command": "tar -C backend --strip 0 -xf", + "exclude_files": [], + "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/nuls.sh --port {{.Ports.BackendRPC}} --dataDir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --pid-file /run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", + "postinst_script_template": "", + "service_type": "forking", + "service_additional_params_template": "", + "protect_memory": false, + "mainnet": true, + "server_config_file": "bitcoin_like.conf", + "client_config_file": "bitcoin_like_client.conf", + "additional_params": { + "addnode": [ + "mainnet.nuls" + ] + } + }, + "blockbook": { + "package_name": "blockbook-nuls", + "system_user": "blockbook-nuls", + "internal_binding_template": ":{{.Ports.BlockbookInternal}}", + "public_binding_template": ":{{.Ports.BlockbookPublic}}", + "explorer_url": "", + "additional_params": "-resyncindexperiod=11000", + "block_chain": { + "parse": true, + "mempool_workers": 4, + "mempool_sub_workers": 8, + "block_addresses_to_keep": 300, + "xpub_magic": 76067358, + "slip44": 133, + "additional_params": {} + } + }, + "meta": { + "package_maintainer": "NULS Core Team", + "package_maintainer_email": "ln@nuls.io" + } +} \ No newline at end of file diff --git a/tests/rpc/testdata/nuls.json b/tests/rpc/testdata/nuls.json new file mode 100644 index 00000000..4ecf423f --- /dev/null +++ b/tests/rpc/testdata/nuls.json @@ -0,0 +1,174 @@ +{ + "blockHeight": 2092530, + "blockHash": "002038a2301c19dec173b6776369f4985a66a4c583c7fa26ae9db7a4369643e71370", + "blockTime": 1552392940, + "blockSize": 290, + "blockTxs": [ + "0020d682b3832334ed03177a7c51a0d94c6cdaee313d14e147203d55ed0f66202234", + "0020ad9d437d9867c778c39a04c030fe941c7c07d9529cf534935471c7c8703ee7c7" + ], + "txDetails": { + "0020d682b3832334ed03177a7c51a0d94c6cdaee313d14e147203d55ed0f66202234": { + "hex": "0100e069d471690100ffffffff000c170423015b98f40fb24b34d395a774aa5ea939c89999a7679e0bff0d00000000daf11f00000017042301a1bdaf958e76c84ff8a17189a6f1f4269ad4a54b7a050f0000000000daf11f00000017042301e60bf29198ed95d53013fdae781f1272a6ccdbda90d70e0000000000daf11f00000017042301751c960ae6c2f9052a53878c9e15f08ad4e79774accb130000000000daf11f00000017042301f69e4b0348458a4846f06dc4940fc1393edd3fcfea7e0e0000000000daf11f000000170423012cfbf9441677fcea3af313da99c01b485e117e9d4f5e0c0000000000daf11f00000017042301d6d33afa523fc78011aafc0c60e2e8dc45206e5676610c0000000000daf11f00000017042301e9e760b295568f955526b29e0d3a0c1dd3a1c50a4f5e0c0000000000daf11f000000170423017e8d9ff74a624f5ee2ef1a9d665385b7314c73a1750c0d0000000000daf11f000000170423012a3435e83852742f65574f297be94c48d9ab68fbb535350100000000daf11f000000170423017904ac680eb1b6a37652fe7fbc57d545d1447d48e55f0c0000000000daf11f00000017042301d2b1c26cdcb86896890bbb741e32d8057ad78cff9dbc180000000000daf11f00000000", + "txid": "0020d682b3832334ed03177a7c51a0d94c6cdaee313d14e147203d55ed0f66202234", + "blocktime": 1552392940, + "time": 1552392940, + "locktime": 2093530, + "version": 0, + "vin": [], + "vout": [ + { + "value": 2.34818462, + "n": 0, + "scriptPubKey": { + "hex": "NsdykbfjmZVHYNaVrKVF89UzUgPKaRa9", + "Addresses": ["NsdykbfjmZVHYNaVrKVF89UzUgPKaRa9"] + } + }, + { + "value": 0.00984442, + "n": 1, + "scriptPubKey": { + "hex": "Nse44mP6EJWbv9qPEPgi4sjtx8QmLzjU", + "Addresses": ["Nse44mP6EJWbv9qPEPgi4sjtx8QmLzjU"] + } + }, + { + "value": 0.00972688, + "n": 2, + "scriptPubKey": { + "hex": "Nse8GNwGtoBhxGmwfetqnhnMART4MVGU", + "Addresses": ["Nse8GNwGtoBhxGmwfetqnhnMART4MVGU"] + } + }, + { + "value": 0.01297324, + "n": 3, + "scriptPubKey": { + "hex": "Nse1KbUNjc8mEGAHgCRHXRWYR6zFa9dh", + "Addresses": ["Nse1KbUNjc8mEGAHgCRHXRWYR6zFa9dh"] + } + }, + { + "value": 0.00949994, + "n": 4, + "scriptPubKey": { + "hex": "Nse9HUun2HwFAypdpBkbjXbQPMkM7nQD", + "Addresses": ["Nse9HUun2HwFAypdpBkbjXbQPMkM7nQD"] + } + }, + { + "value": 0.00810575, + "n": 5, + "scriptPubKey": { + "hex": "NsdvtMNdncJC3q8un2DpBWacZL6YRzJp", + "Addresses": ["NsdvtMNdncJC3q8un2DpBWacZL6YRzJp"] + } + }, + { + "value": 0.00811382, + "n": 6, + "scriptPubKey": { + "hex": "Nse7L6F3sEGc5NRghLtkRrVGaxwgXePS", + "Addresses": ["Nse7L6F3sEGc5NRghLtkRrVGaxwgXePS"] + } + }, + { + "value": 0.00810575, + "n": 7, + "scriptPubKey": { + "hex": "Nse8W8pMG1E94HQSkieUYUyASBr6oRQr", + "Addresses": ["Nse8W8pMG1E94HQSkieUYUyASBr6oRQr"] + } + }, + { + "value": 0.00855157, + "n": 8, + "scriptPubKey": { + "hex": "Nse1uGXjbCm9JJjmL88xYhnU59B6b3GR", + "Addresses": ["Nse1uGXjbCm9JJjmL88xYhnU59B6b3GR"] + } + }, + { + "value": 0.20264373, + "n": 9, + "scriptPubKey": { + "hex": "NsdviSF29wA91UtGgYuN3HL6iMWLs2hc", + "Addresses": ["NsdviSF29wA91UtGgYuN3HL6iMWLs2hc"] + } + }, + { + "value": 0.00810981, + "n": 10, + "scriptPubKey": { + "hex": "Nse1ZXaeJDcuteKwbxacdBnX46bzfcuQ", + "Addresses": ["Nse1ZXaeJDcuteKwbxacdBnX46bzfcuQ"] + } + }, + { + "value": 0.01621149, + "n": 11, + "scriptPubKey": { + "hex": "Nse75MmS9stKYE11KCaLaVRaXHLprikS", + "Addresses": ["Nse75MmS9stKYE11KCaLaVRaXHLprikS"] + } + } + ] + }, + "0020ad9d437d9867c778c39a04c030fe941c7c07d9529cf534935471c7c8703ee7c7": { + "hex": "02004a53d471690100ffffffff03230020199146789cda517776c140a8638218b8fa29e3a1857573f9463df3e6fa1f56100167537e3b680000000000000000002300208fa6362e79b40b61791dd1484b79d61c9bebf4dc4986f347ba46e65feb9ee4dc00e02c504d00000000000000000000230020fe3ca565e667bd6b2b7889c1307d2006d0ce5697d13e98311c605ca802c7ef0f00603924cb2e0200000000000000000217042301405aa9eeca8af1edae196837bde4eef29bee040600664280d1000000000000000000170423016056a17c02cd53191ce2e647c5ed02d1c70e483607cdaed3c50100000000000000006b2102f23b91b10ca411dceccf873a313399f0c3a90b14e7b0b5f43642e33da882b7bc00473045022100c2521e5951d168e43f42010e8f1ba484cb445dcdb29d066caf0bce1c78476624022074df5ccdd4eda6f3adec5420efef85d7369b9b8266bbc392f5e76eef94f0b9a9", + "txid": "0020ad9d437d9867c778c39a04c030fe941c7c07d9529cf534935471c7c8703ee7c7", + "blocktime": 1552392940, + "time": 1552392934, + "locktime": 0, + "version": 0, + "vin": [ + { + "txid": "0020199146789cda517776c140a8638218b8fa29e3a1857573f9463df3e6fa1f5610", + "vout": 1, + "sequence": 0, + "scriptSig": { + "hex": "" + }, + "Addresses": ["Nsdz3WNA1bfGSmy5oYY3KGMjZSd1C61p"] + }, + { + "txid": "00208fa6362e79b40b61791dd1484b79d61c9bebf4dc4986f347ba46e65feb9ee4dc", + "vout": 0, + "sequence": 0, + "scriptSig": { + "hex": "" + }, + "Addresses": ["Nsdz3WNA1bfGSmy5oYY3KGMjZSd1C61p"] + }, + { + "txid": "0020fe3ca565e667bd6b2b7889c1307d2006d0ce5697d13e98311c605ca802c7ef0f", + "vout": 0, + "sequence": 0, + "scriptSig": { + "hex": "" + }, + "Addresses": ["Nsdz3WNA1bfGSmy5oYY3KGMjZSd1C61p"] + } + ], + "vout": [ + { + "value": 8998.00000000, + "n": 0, + "scriptPubKey": { + "hex": "Nsdx5SBU9xN9SKz6dEwcz85WdFaNj2g4", + "Addresses": ["Nsdx5SBU9xN9SKz6dEwcz85WdFaNj2g4"] + } + }, + { + "value": 19491.71633415, + "n": 1, + "scriptPubKey": { + "hex": "Nsdz3WNA1bfGSmy5oYY3KGMjZSd1C61p", + "Addresses": ["Nsdz3WNA1bfGSmy5oYY3KGMjZSd1C61p"] + } + } + ] + } + } +} diff --git a/tests/sync/testdata/nuls.json b/tests/sync/testdata/nuls.json new file mode 100644 index 00000000..63579dcb --- /dev/null +++ b/tests/sync/testdata/nuls.json @@ -0,0 +1,546 @@ +{ + "connectBlocks": { + "syncRanges": [ + {"lower": 2184003, "upper": 2184213} + ], + "blocks": { + "2184003": { + "height": 2184003, + "hash": "0020e281a2e5095904f95e3d1f3b48da808c7511e478362f22e678bf68b976e1bbb6", + "noTxs": 1, + "txDetails": [ + { + "hex": "", + "txid": "00208f0b652c397e761e0e2e1fc31f8b1b4f7fe28a1f35af382886c4d95c8dcb9715", + "blocktime": 1553317770, + "time": 1553317770, + "version": 0, + "vin": [], + "vout": [ + { + "value": 0.21957008, + "n": 0, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "NsdteeXfqPcsmMStKeE2Es3dv7rSXjgF", + "Addresses": ["NsdteeXfqPcsmMStKeE2Es3dv7rSXjgF"] + } + }, + { + "value": 0.00836727, + "n": 1, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse4H3MD4thD2rZeW178J6AxUh4B5Dq3", + "Addresses": ["Nse4H3MD4thD2rZeW178J6AxUh4B5Dq3"] + } + }, + { + "value": 0.00823145, + "n": 2, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nsdx6yBvJRtguGduzxkgxEPjkMebW9DC", + "Addresses": ["Nsdx6yBvJRtguGduzxkgxEPjkMebW9DC"] + } + }, + { + "value": 0.00992713, + "n": 3, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse72BM97n88MFyhWDp9iDPEbkw4fqVh", + "Addresses": ["Nse72BM97n88MFyhWDp9iDPEbkw4fqVh"] + } + }, + { + "value": 0.41158111, + "n": 4, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse2c2siYena4BvSn82iprugFJE4gUsJ", + "Addresses": ["Nse2c2siYena4BvSn82iprugFJE4gUsJ"] + } + }, + { + "value": 0.00823145, + "n": 5, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "NsdxARm7PHjiPLgnhMMbXLR2n7R2eded", + "Addresses": ["NsdxARm7PHjiPLgnhMMbXLR2n7R2eded"] + } + },{ + "value": 0.01243773, + "n": 6, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "NsdycgzrFdoev6nmxxMGJvXKTYzo36vj", + "Addresses": ["NsdycgzrFdoev6nmxxMGJvXKTYzo36vj"] + } + }, + { + "value": 0.01009588, + "n": 7, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse5p6NULbecLCe8mZVVzBcSMpaZy2bB", + "Addresses": ["Nse5p6NULbecLCe8mZVVzBcSMpaZy2bB"] + } + },{ + "value": 0.00823145, + "n": 8, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse4e7QYA3Mj9UUqcGpCbYKexwgtSCvz", + "Addresses": ["Nse4e7QYA3Mj9UUqcGpCbYKexwgtSCvz"] + } + }, + { + "value": 0.00823145, + "n": 9, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "NsdyL8HuKgrQehmefzZgKo2aydfRJQSU", + "Addresses": ["NsdyL8HuKgrQehmefzZgKo2aydfRJQSU"] + } + }, + { + "value": 0.01646950, + "n": 10, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse8jHzR4cS1iyiJkewGHKEWqm5B7ZbA", + "Addresses": ["Nse8jHzR4cS1iyiJkewGHKEWqm5B7ZbA"] + } + }, + { + "value": 0.00823145, + "n": 11, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "NsdveDWtHKGCbxVH1AKkMA2oB2VUv134", + "Addresses": ["NsdveDWtHKGCbxVH1AKkMA2oB2VUv134"] + } + }, + { + "value": 0.01704543, + "n": 12, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nsdz1QdJb7xNE4bDNg13oXMfa2jtbh36", + "Addresses": ["Nsdz1QdJb7xNE4bDNg13oXMfa2jtbh36"] + } + }, + { + "value": 0.03778238, + "n": 13, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse4rPNt3S3HHZaRhuXRDfVFYSKA4TzR", + "Addresses": ["Nse4rPNt3S3HHZaRhuXRDfVFYSKA4TzR"] + } + }, + { + "value": 0.41155642, + "n": 14, + "lockTime": 2185003, + "scriptPubKey": { + "hex": "Nse84iqmZipKg7Cn6LtV5uQshWwtdRMJ", + "Addresses": ["Nse84iqmZipKg7Cn6LtV5uQshWwtdRMJ"] + } + } + ] + } + ] + }, + "2184004": { + "height": 2184004, + "hash": "002099f701eb56f8ff3486d10a4e981df04791bcc47f6749d10fbec0987bf425e8d7", + "noTxs": 1, + "txDetails": [ + { + "hex": "", + "txid": "0020ecd57d05aa56098114d2284cf48ab6edceab2351aa43add3af09b3a956fb458e", + "blocktime": 1553317780, + "time": 1553317780, + "version": 0, + "vin": [], + "vout": [ + { + "value": 2.39921315, + "n": 0, + "lockTime": 2185004, + "scriptPubKey": { + "hex": "NsdwtSWZrG5gwUDoR7VfxxCVzxYsNm3Y", + "Addresses": ["NsdwtSWZrG5gwUDoR7VfxxCVzxYsNm3Y"] + } + } + ] + } + ] + }, + "2184005": { + "height": 2184005, + "hash": "00203f9e06e59bda071f7af9fef59a53604e59a6640c52635f6e18101396b616de82", + "noTxs": 1, + "txDetails": [ + { + "hex": "", + "txid": "002011a00b4561f8bb278d8e4744550368b217d61013fcd1eebc61f44a1c6a164765", + "blocktime": 1553317790, + "time": 1553317790, + "version": 0, + "vin": [], + "vout": [ + { + "value": 0.57503254, + "n": 0, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nsduq2g457guft4bpj44BVFqVow6kyWi", + "Addresses": ["Nsduq2g457guft4bpj44BVFqVow6kyWi"] + } + }, + { + "value": 0.04152482, + "n": 1, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "NsdvZbRf6pY1ggVfwfBR8DVs4KwUyd7L", + "Addresses": ["NsdvZbRf6pY1ggVfwfBR8DVs4KwUyd7L"] + } + }, + { + "value": 0.02018105, + "n": 2, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse7uEeE7aRrhFyio9ZLa2wpx7aZ6oBf", + "Addresses": ["Nse7uEeE7aRrhFyio9ZLa2wpx7aZ6oBf"] + } + }, + { + "value": 0.01177228, + "n": 3, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse2EYyQXm5EntkGGNU1omz4AZVVXRyP", + "Addresses": ["Nse2EYyQXm5EntkGGNU1omz4AZVVXRyP"] + } + }, + { + "value": 0.02932482, + "n": 4, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "NsdyewTUzSVwyyk55oL4WA7mR3tEt7cb", + "Addresses": ["NsdyewTUzSVwyyk55oL4WA7mR3tEt7cb"] + } + }, + { + "value": 0.08304964, + "n": 5, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "NsdtpVCa8nMtcZoWcvBDDKdmvvvQiqB4", + "Addresses": ["NsdtpVCa8nMtcZoWcvBDDKdmvvvQiqB4"] + } + },{ + "value": 0.16040263, + "n": 6, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "NsdupfMS5mSvxTquqKdAtzzqrbzGrEVX", + "Addresses": ["NsdupfMS5mSvxTquqKdAtzzqrbzGrEVX"] + } + }, + { + "value": 0.02906321, + "n": 7, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nsdya19a18roRTWLKNZ4ivsswRLcQq3m", + "Addresses": ["Nsdya19a18roRTWLKNZ4ivsswRLcQq3m"] + } + },{ + "value": 0.02074533, + "n": 8, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse3mU18MXLYTuGGvCXyk3U1Zmf3d129", + "Addresses": ["Nse3mU18MXLYTuGGvCXyk3U1Zmf3d129"] + } + }, + { + "value": 0.04303632, + "n": 9, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse11hHcyLS9pidxowEB6Nupf6rp1y83", + "Addresses": ["Nse11hHcyLS9pidxowEB6Nupf6rp1y83"] + } + }, + { + "value": 0.24783259, + "n": 10, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse3rftMFxSy3vhRsyyK1axLrD1f4gvF", + "Addresses": ["Nse3rftMFxSy3vhRsyyK1axLrD1f4gvF"] + } + }, + { + "value": 0.03612242, + "n": 11, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse8aukpCpwW43qnogQN3WV2SHjbgUeU", + "Addresses": ["Nse8aukpCpwW43qnogQN3WV2SHjbgUeU"] + } + }, + { + "value": 0.09346822, + "n": 12, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse31sLCY9iDtsU1Fb5nwMzHQx1SC2pW", + "Addresses": ["Nse31sLCY9iDtsU1Fb5nwMzHQx1SC2pW"] + } + }, + { + "value": 0.04232209, + "n": 13, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse3heixrSj8Ad2iYymSqZLEASEx46ct", + "Addresses": ["Nse3heixrSj8Ad2iYymSqZLEASEx46ct"] + } + }, + { + "value": 0.22471157, + "n": 14, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse16kQr7z2msh5gerMbPDCNyZJFFgui", + "Addresses": ["Nse16kQr7z2msh5gerMbPDCNyZJFFgui"] + } + }, + { + "value": 0.02124410, + "n": 15, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "NsdwqY7WsekKmzmwdbgkack2VZMo1ehq", + "Addresses": ["NsdwqY7WsekKmzmwdbgkack2VZMo1ehq"] + } + }, + { + "value": 0.04152482, + "n": 16, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nsdzsz17JDYQbEmZReG8vqveoXhRXVqL", + "Addresses": ["Nsdzsz17JDYQbEmZReG8vqveoXhRXVqL"] + } + }, + { + "value": 0.20762411, + "n": 17, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse2rtdaQLBPiyoVmJH4BQHJn2VUSHjo", + "Addresses": ["Nse2rtdaQLBPiyoVmJH4BQHJn2VUSHjo"] + } + }, + { + "value": 0.09136708, + "n": 18, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse7W2dvVKFT2UYoavU4vVFeSYG9rjb1", + "Addresses": ["Nse7W2dvVKFT2UYoavU4vVFeSYG9rjb1"] + } + }, + { + "value": 0.03410019, + "n": 19, + "lockTime": 2185005, + "scriptPubKey": { + "hex": "Nse1PbFcq9TJirXsvsjMooZ5RM1A8y7C", + "Addresses": ["Nse1PbFcq9TJirXsvsjMooZ5RM1A8y7C"] + } + } + ] + } + ] + }, + "2184031": { + "height": 2184031, + "hash": "00202ef14e5a5505e8b8658af6c15110c1b9cf4f5dead3cfb363bbaa5ffb2275ee6e", + "noTxs": 2, + "txDetails": [ + { + "hex": "", + "txid": "00206d1d22475202bbb81faba6b0e9ba22fe35c7e9d1e542d3284c5002c5a2ef7ca5", + "blocktime": 1553318050, + "time": 1553318050, + "version": 0, + "vin": [], + "vout": [ + { + "value": 1.14055774, + "n": 0, + "lockTime": 2185031, + "scriptPubKey": { + "hex": "Nse5cCyitYFJ7j6ncn5UCst7pc672QEL", + "Addresses": ["Nse5cCyitYFJ7j6ncn5UCst7pc672QEL"] + } + }, + { + "value": 0.02054451, + "n": 1, + "lockTime": 2185031, + "scriptPubKey": { + "hex": "Nse24Dta8UYJtR6jZHSqWtrfdRpnfHYd", + "Addresses": ["Nse24Dta8UYJtR6jZHSqWtrfdRpnfHYd"] + } + }, + { + "value": 0.35934295, + "n": 2, + "lockTime": 2185031, + "scriptPubKey": { + "hex": "Nse5S92swYm231iDVZNTrDurMcGCWBcM", + "Addresses": ["Nse5S92swYm231iDVZNTrDurMcGCWBcM"] + } + }, + { + "value": 0.38161210, + "n": 3, + "lockTime": 2185031, + "scriptPubKey": { + "hex": "NsdviSF29wA91UtGgYuN3HL6iMWLs2hc", + "Addresses": ["NsdviSF29wA91UtGgYuN3HL6iMWLs2hc"] + } + }, + { + "value": 0.32403928, + "n": 4, + "lockTime": 2185031, + "scriptPubKey": { + "hex": "Nse8nchECsiWQ2eyTgzjvH4DewAWqvpR", + "Addresses": ["Nse8nchECsiWQ2eyTgzjvH4DewAWqvpR"] + } + } + ] + }, + { + "hex": "", + "txid": "0020643416a8b1dad09ee2c1bb2d12aa88eef77a6462a6bd16c39fd3971d67d66919", + "blocktime": 1553318050, + "time": 1553318045, + "version": 0, + "vin": [ + { + "txid": "00205462276c5fb51b465c3926dceb27a6a724e63410c146bb027fdd7f7f06bcb56f", + "vout": 0, + "sequence": 0, + "scriptSig": { + "hex": "" + } + } + ], + "vout": [ + { + "value": 5444.999, + "n": 0, + "lockTime": 0, + "scriptPubKey": { + "hex": "Nsdwnd4auFisFJKU6iDvBxTdPkeg8qkB", + "Addresses": ["Nsdwnd4auFisFJKU6iDvBxTdPkeg8qkB"] + } + } + ] + } + ] + }, + "2184213": { + "height": 2184213, + "hash": "002052a452263860be5bba31a3aace06c1cb524d48cda3d04575c3881b10241cca06", + "noTxs": 1, + "txDetails": [ + { + "hex": "", + "txid": "00204882a15d826d4401aa765510243f455f7f6a05b0e77a9edd420d366cabd98818", + "blocktime": 1553319870, + "time": 1553319870, + "version": 0, + "vin": [], + "vout": [ + { + "value": 0.04070938, + "n": 0, + "lockTime": 2185213, + "scriptPubKey": { + "hex": "NsdwTB38bAwybyKTfUcxZoLGEHSVqpNE", + "Addresses": ["NsdwTB38bAwybyKTfUcxZoLGEHSVqpNE"] + } + }, + { + "value": 0.20863453, + "n": 1, + "lockTime": 2185213, + "scriptPubKey": { + "hex": "Nse8Q7gk2gVGDdHk7rxasedRRjgVc18Q", + "Addresses": ["Nse8Q7gk2gVGDdHk7rxasedRRjgVc18Q"] + } + } + ] + } + ] + } + } + }, + "handleFork": { + "syncRanges": [ + {"lower": 2184210, "upper": 2184213} + ], + "fakeBlocks": { + "2184210": { + "height": 2184210, + "hash": "00205087cb3e6964a51b2be17593660ed4fa460706742d0d97092541d604b7ef4eaa" + }, + "2184211": { + "height": 2184211, + "hash": "00204ba087c1ab04a45cfef6c61c16f97b72d118dc08fc627d7216b4a38257e906c4" + }, + "2184212": { + "height": 2184212, + "hash": "002029540932d67260a4dbfb7796ffe2a96bf3abbd5c4898814b8a3057c5392d0cbb" + }, + "2184213": { + "height": 2184213, + "hash": "00209d3d7e5869f1f32d4bd75587691f426d21e28022e98bd7ef0ca76590c8a6eb76" + } + }, + "realBlocks": { + "2184210": { + "height": 2184210, + "hash": "00200fc994cceb3aa75a9adbdbfa5d2b9dbfaddb8ca88f1cd90acc43e0e775abfff3" + }, + "2184211": { + "height": 2184211, + "hash": "0020a3d90cf5962177a775f47d2edaed94b2519d0cde93711d5e95878c9c6d390b85" + }, + "2184212": { + "height": 2184212, + "hash": "002027d151738f460437db0ce8aaa058cd5fa9230a4e59e5257cc53e059f05419c3c" + }, + "2184213": { + "height": 2184213, + "hash": "002052a452263860be5bba31a3aace06c1cb524d48cda3d04575c3881b10241cca06" + } + } + } +} diff --git a/tests/tests.json b/tests/tests.json index df34f0b4..82a45ccb 100644 --- a/tests/tests.json +++ b/tests/tests.json @@ -125,5 +125,9 @@ "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateSmartFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"], "sync": ["ConnectBlocksParallel", "ConnectBlocks", "HandleFork"] + }, + "nuls": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"], + "sync": ["ConnectBlocksParallel", "ConnectBlocks", "HandleFork"] } } From a8735c460d6da5597a30d950b5befed6d8829413 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 28 Mar 2019 14:32:33 +0100 Subject: [PATCH 11/39] Update ports registry --- docs/ports.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ports.md b/docs/ports.md index 51c4b72b..91f5c727 100644 --- a/docs/ports.md +++ b/docs/ports.md @@ -25,6 +25,7 @@ | Zcoin | 9050 | 9150 | 8050 | 38350 | | Koto | 9051 | 9151 | 8051 | 38351 | | Bellcoin | 9052 | 9152 | 8052 | 38352 | +| NULS | 9053 | 9153 | 8053 | 38353 | | Flo | 9066 | 9166 | 8066 | 38366 | | Qtum | 9088 | 9188 | 8088 | 38388 | | Bitcoin Testnet | 19030 | 19130 | 18030 | 48330 | From d2928b3516ee99cb28c74b67d400a4826c6cdb9e Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 27 Mar 2019 12:46:45 +0100 Subject: [PATCH 12/39] Refactor server init in program startup --- blockbook.go | 124 +++++++++++++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 54 deletions(-) diff --git a/blockbook.go b/blockbook.go index 6315ad52..f5b62377 100644 --- a/blockbook.go +++ b/blockbook.go @@ -99,30 +99,6 @@ func init() { glog.CopyStandardLogTo("INFO") } -func getBlockChainWithRetry(coin string, configfile string, pushHandler func(bchain.NotificationType), metrics *common.Metrics, seconds int) (bchain.BlockChain, bchain.Mempool, error) { - var chain bchain.BlockChain - var mempool bchain.Mempool - var err error - timer := time.NewTimer(time.Second) - for i := 0; ; i++ { - if chain, mempool, err = coins.NewBlockChain(coin, configfile, pushHandler, metrics); err != nil { - if i < seconds { - glog.Error("rpc: ", err, " Retrying...") - select { - case <-chanOsSignal: - return nil, nil, errors.New("Interrupted") - case <-timer.C: - timer.Reset(time.Second) - continue - } - } else { - return nil, nil, err - } - } - return chain, mempool, nil - } -} - func main() { flag.Parse() @@ -227,45 +203,20 @@ func main() { var internalServer *server.InternalServer if *internalBinding != "" { - internalServer, err = server.NewInternalServer(*internalBinding, *certFiles, index, chain, mempool, txCache, internalState) + internalServer, err = startInternalServer() if err != nil { - glog.Error("https: ", err) + glog.Error("internal server: ", err) return } - go func() { - err = internalServer.Run() - if err != nil { - if err.Error() == "http: Server closed" { - glog.Info("internal server: closed") - } else { - glog.Error(err) - return - } - } - }() } var publicServer *server.PublicServer if *publicBinding != "" { - // start public server in limited functionality, extend it after sync is finished by calling ConnectFullPublicInterface - publicServer, err = server.NewPublicServer(*publicBinding, *certFiles, index, chain, mempool, txCache, *explorerURL, metrics, internalState, *debugMode) + publicServer, err = startPublicServer() if err != nil { - glog.Error("socketio: ", err) + glog.Error("public server: ", err) return } - go func() { - err = publicServer.Run() - if err != nil { - if err.Error() == "http: Server closed" { - glog.Info("public server: closed") - } else { - glog.Error(err) - return - } - } - }() - callbacksOnNewBlock = append(callbacksOnNewBlock, publicServer.OnNewBlock) - callbacksOnNewTxAddr = append(callbacksOnNewTxAddr, publicServer.OnNewTxAddr) } if *synchronize { @@ -293,7 +244,7 @@ func main() { } go storeInternalStateLoop() - if *publicBinding != "" { + if publicServer != nil { // start full public interface publicServer.ConnectFullPublicInterface() } @@ -327,6 +278,71 @@ func main() { } } +func getBlockChainWithRetry(coin string, configfile string, pushHandler func(bchain.NotificationType), metrics *common.Metrics, seconds int) (bchain.BlockChain, bchain.Mempool, error) { + var chain bchain.BlockChain + var mempool bchain.Mempool + var err error + timer := time.NewTimer(time.Second) + for i := 0; ; i++ { + if chain, mempool, err = coins.NewBlockChain(coin, configfile, pushHandler, metrics); err != nil { + if i < seconds { + glog.Error("rpc: ", err, " Retrying...") + select { + case <-chanOsSignal: + return nil, nil, errors.New("Interrupted") + case <-timer.C: + timer.Reset(time.Second) + continue + } + } else { + return nil, nil, err + } + } + return chain, mempool, nil + } +} + +func startInternalServer() (*server.InternalServer, error) { + internalServer, err := server.NewInternalServer(*internalBinding, *certFiles, index, chain, mempool, txCache, internalState) + if err != nil { + return nil, err + } + go func() { + err = internalServer.Run() + if err != nil { + if err.Error() == "http: Server closed" { + glog.Info("internal server: closed") + } else { + glog.Error(err) + return + } + } + }() + return internalServer, nil +} + +func startPublicServer() (*server.PublicServer, error) { + // start public server in limited functionality, extend it after sync is finished by calling ConnectFullPublicInterface + publicServer, err := server.NewPublicServer(*publicBinding, *certFiles, index, chain, mempool, txCache, *explorerURL, metrics, internalState, *debugMode) + if err != nil { + return nil, err + } + go func() { + err = publicServer.Run() + if err != nil { + if err.Error() == "http: Server closed" { + glog.Info("public server: closed") + } else { + glog.Error(err) + return + } + } + }() + callbacksOnNewBlock = append(callbacksOnNewBlock, publicServer.OnNewBlock) + callbacksOnNewTxAddr = append(callbacksOnNewTxAddr, publicServer.OnNewTxAddr) + return publicServer, err +} + func performRollback() { bestHeight, bestHash, err := index.GetBestBlock() if err != nil { From c813f763369b2c4c1e30d389014de3fbb64d5ea6 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 29 Mar 2019 17:01:20 +0100 Subject: [PATCH 13/39] Try to load mempool inputs from db to speed up mempool sync --- bchain/coins/blockchain.go | 4 +-- bchain/coins/btc/bitcoinrpc.go | 8 ++++-- bchain/coins/eth/ethrpc.go | 5 +++- bchain/coins/nuls/nulsrpc.go | 5 ++-- bchain/mempool_bitcoin_type.go | 45 ++++++++++++++++++++-------------- bchain/types.go | 7 ++++-- blockbook.go | 6 ++++- db/rocksdb.go | 19 ++++++++++++++ tests/dbtestdata/fakechain.go | 2 +- tests/integration.go | 2 +- 10 files changed, 72 insertions(+), 31 deletions(-) diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 6d1cdcb6..624b5d75 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -152,8 +152,8 @@ func (c *blockChainWithMetrics) CreateMempool() (bchain.Mempool, error) { return c.b.CreateMempool() } -func (c *blockChainWithMetrics) InitializeMempool() error { - return c.b.InitializeMempool() +func (c *blockChainWithMetrics) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { + return c.b.InitializeMempool(addrDescForOutpoint) } func (c *blockChainWithMetrics) Shutdown(ctx context.Context) error { diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index f26830d2..a6c4cdf0 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -138,8 +138,12 @@ func (b *BitcoinRPC) CreateMempool() (bchain.Mempool, error) { return b.Mempool, nil } -// InitializeMempool creates ZeroMQ subscription -func (b *BitcoinRPC) InitializeMempool() error { +// InitializeMempool creates ZeroMQ subscription and sets AddrDescForOutpointFunc to the Mempool +func (b *BitcoinRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { + if b.Mempool == nil { + return errors.New("Mempool not created") + } + b.Mempool.AddrDescForOutpoint = addrDescForOutpoint if b.mq == nil { mq, err := bchain.NewMQ(b.ChainConfig.MessageQueueBinding, b.pushHandler) if err != nil { diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index bac05d97..18643370 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -172,7 +172,10 @@ func (b *EthereumRPC) CreateMempool() (bchain.Mempool, error) { } // InitializeMempool creates subscriptions to newHeads and newPendingTransactions -func (b *EthereumRPC) InitializeMempool() error { +func (b *EthereumRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { + if b.Mempool == nil { + return errors.New("Mempool not created") + } if b.isETC { glog.Info(b.ChainConfig.CoinName, " does not support subscription to newHeads") } else { diff --git a/bchain/coins/nuls/nulsrpc.go b/bchain/coins/nuls/nulsrpc.go index 3bd7b24e..49e4178f 100644 --- a/bchain/coins/nuls/nulsrpc.go +++ b/bchain/coins/nuls/nulsrpc.go @@ -5,9 +5,8 @@ import ( "blockbook/bchain/coins/btc" "bytes" "encoding/base64" + "encoding/hex" "encoding/json" - "github.com/gobuffalo/packr/v2/file/resolver/encoding/hex" - "github.com/juju/errors" "io" "io/ioutil" "math/big" @@ -17,6 +16,8 @@ import ( "strconv" "time" + "github.com/juju/errors" + "github.com/golang/glog" ) diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index e87cc937..ae13459e 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -19,13 +19,14 @@ type txidio struct { // MempoolBitcoinType is mempool handle. type MempoolBitcoinType struct { - chain BlockChain - mux sync.Mutex - txToInputOutput map[string][]addrIndex - addrDescToTx map[string][]Outpoint - chanTxid chan string - chanAddrIndex chan txidio - onNewTxAddr OnNewTxAddrFunc + chain BlockChain + mux sync.Mutex + txToInputOutput map[string][]addrIndex + addrDescToTx map[string][]Outpoint + chanTxid chan string + chanAddrIndex chan txidio + onNewTxAddr OnNewTxAddrFunc + AddrDescForOutpoint AddrDescForOutpointFunc } // NewMempoolBitcoinType creates new mempool handler. @@ -86,19 +87,25 @@ func (m *MempoolBitcoinType) updateMappings(newTxToInputOutput map[string][]addr } func (m *MempoolBitcoinType) getInputAddress(input Outpoint) *addrIndex { - itx, err := m.chain.GetTransactionForMempool(input.Txid) - if err != nil { - glog.Error("cannot get transaction ", input.Txid, ": ", err) - return nil + var addrDesc AddressDescriptor + if m.AddrDescForOutpoint != nil { + addrDesc = m.AddrDescForOutpoint(input) } - if int(input.Vout) >= len(itx.Vout) { - glog.Error("Vout len in transaction ", input.Txid, " ", len(itx.Vout), " input.Vout=", input.Vout) - return nil - } - addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&itx.Vout[input.Vout]) - if err != nil { - glog.Error("error in addrDesc in ", input.Txid, " ", input.Vout, ": ", err) - return nil + if addrDesc == nil { + itx, err := m.chain.GetTransactionForMempool(input.Txid) + if err != nil { + glog.Error("cannot get transaction ", input.Txid, ": ", err) + return nil + } + if int(input.Vout) >= len(itx.Vout) { + glog.Error("Vout len in transaction ", input.Txid, " ", len(itx.Vout), " input.Vout=", input.Vout) + return nil + } + addrDesc, err = m.chain.GetChainParser().GetAddrDescFromVout(&itx.Vout[input.Vout]) + if err != nil { + glog.Error("error in addrDesc in ", input.Txid, " ", input.Vout, ": ", err) + return nil + } } return &addrIndex{string(addrDesc), ^input.Vout} diff --git a/bchain/types.go b/bchain/types.go index 044122e6..0415f4b8 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -191,15 +191,18 @@ type OnNewBlockFunc func(hash string, height uint32) // OnNewTxAddrFunc is used to send notification about a new transaction/address type OnNewTxAddrFunc func(tx *Tx, desc AddressDescriptor) +// AddrDescForOutpointFunc defines function that returns address descriptorfor given outpoint or nil if outpoint not found +type AddrDescForOutpointFunc func(outpoint Outpoint) AddressDescriptor + // BlockChain defines common interface to block chain daemon type BlockChain interface { // life-cycle methods - // intialize the block chain connector + // initialize the block chain connector Initialize() error // create mempool but do not initialize it CreateMempool() (Mempool, error) // initialize mempool, create ZeroMQ (or other) subscription - InitializeMempool() error + InitializeMempool(AddrDescForOutpointFunc) error // shutdown mempool, ZeroMQ and block chain connections Shutdown(ctx context.Context) error // chain info diff --git a/blockbook.go b/blockbook.go index f5b62377..e694bcc2 100644 --- a/blockbook.go +++ b/blockbook.go @@ -227,7 +227,11 @@ func main() { return } // initialize mempool after the initial sync is complete - err = chain.InitializeMempool() + var addrDescForOutpoint bchain.AddrDescForOutpointFunc + if chain.GetChainParser().GetChainType() == bchain.ChainBitcoinType { + addrDescForOutpoint = index.AddrDescForOutpoint + } + err = chain.InitializeMempool(addrDescForOutpoint) if err != nil { glog.Error("initializeMempool ", err) return diff --git a/db/rocksdb.go b/db/rocksdb.go index 0ec55b03..735402e3 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -757,6 +757,25 @@ func (d *RocksDB) GetTxAddresses(txid string) (*TxAddresses, error) { return d.getTxAddresses(btxID) } +// AddrDescForOutpoint defines function that returns address descriptorfor given outpoint or nil if outpoint not found +func (d *RocksDB) AddrDescForOutpoint(outpoint bchain.Outpoint) bchain.AddressDescriptor { + ta, err := d.GetTxAddresses(outpoint.Txid) + if err != nil || ta == nil { + return nil + } + if outpoint.Vout < 0 { + vin := ^outpoint.Vout + if len(ta.Inputs) <= int(vin) { + return nil + } + return ta.Inputs[vin].AddrDesc + } + if len(ta.Outputs) <= int(outpoint.Vout) { + return nil + } + return ta.Outputs[outpoint.Vout].AddrDesc +} + func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { buf = buf[:0] l := packVaruint(uint(ta.Height), varBuf) diff --git a/tests/dbtestdata/fakechain.go b/tests/dbtestdata/fakechain.go index 2f05bd4b..270403bb 100644 --- a/tests/dbtestdata/fakechain.go +++ b/tests/dbtestdata/fakechain.go @@ -25,7 +25,7 @@ func (c *fakeBlockChain) Initialize() error { return nil } -func (c *fakeBlockChain) InitializeMempool() error { +func (c *fakeBlockChain) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { return nil } diff --git a/tests/integration.go b/tests/integration.go index 789f71fc..d1e70130 100644 --- a/tests/integration.go +++ b/tests/integration.go @@ -174,7 +174,7 @@ func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, bc return nil, nil, fmt.Errorf("Mempool creation failed: %s", err) } - err = cli.InitializeMempool() + err = cli.InitializeMempool(nil) if err != nil { return nil, nil, fmt.Errorf("Mempool initialization failed: %s", err) } From 827cbcd1d832a81a1559deaadd17aee56fc03327 Mon Sep 17 00:00:00 2001 From: Yura Pakhuchiy Date: Thu, 28 Mar 2019 14:05:43 +0300 Subject: [PATCH 14/39] Support Groestlcoin xpub --- Gopkg.lock | 2 +- bchain/coins/btc/bitcoinparser.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 4bbfcd31..5553540c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -95,7 +95,7 @@ branch = "master" name = "github.com/martinboehm/btcutil" packages = [".","base58","bech32","chaincfg","hdkeychain","txscript"] - revision = "63034958e64b209cb9294128309dbaed497cde7b" + revision = "225ed00dbbd5cb8d8b3949a0ee7c9ea540754585" [[projects]] branch = "master" diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index 6142af51..eed0878c 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -343,7 +343,7 @@ func (p *BitcoinParser) addrDescFromExtKey(extKey *hdkeychain.ExtendedKey) (bcha // DeriveAddressDescriptors derives address descriptors from given xpub for listed indexes func (p *BitcoinParser) DeriveAddressDescriptors(xpub string, change uint32, indexes []uint32) ([]bchain.AddressDescriptor, error) { - extKey, err := hdkeychain.NewKeyFromString(xpub) + extKey, err := hdkeychain.NewKeyFromString(xpub, p.Params.Base58CksumHasher) if err != nil { return nil, err } @@ -370,7 +370,7 @@ func (p *BitcoinParser) DeriveAddressDescriptorsFromTo(xpub string, change uint3 if toIndex <= fromIndex { return nil, errors.New("toIndex<=fromIndex") } - extKey, err := hdkeychain.NewKeyFromString(xpub) + extKey, err := hdkeychain.NewKeyFromString(xpub, p.Params.Base58CksumHasher) if err != nil { return nil, err } @@ -394,7 +394,7 @@ func (p *BitcoinParser) DeriveAddressDescriptorsFromTo(xpub string, change uint3 // DerivationBasePath returns base path of xpub func (p *BitcoinParser) DerivationBasePath(xpub string) (string, error) { - extKey, err := hdkeychain.NewKeyFromString(xpub) + extKey, err := hdkeychain.NewKeyFromString(xpub, p.Params.Base58CksumHasher) if err != nil { return "", err } From f2dc4a56d8c476efd7bf163e34bac6a8de635d21 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 1 Apr 2019 17:00:34 +0200 Subject: [PATCH 15/39] Store time of mempool transaction --- bchain/coins/blockchain.go | 8 +-- bchain/coins/btc/bitcoinrpc.go | 3 +- bchain/coins/eth/ethrpc.go | 3 +- bchain/mempool_bitcoin_type.go | 44 ++++++------ bchain/mempool_ethereum_type.go | 123 +++++++++++++++++--------------- bchain/types.go | 4 +- blockbook.go | 10 +-- tests/dbtestdata/fakechain.go | 2 +- tests/integration.go | 2 +- tests/rpc/rpc.go | 2 +- 10 files changed, 109 insertions(+), 92 deletions(-) diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 624b5d75..6f1913fc 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -152,8 +152,8 @@ func (c *blockChainWithMetrics) CreateMempool() (bchain.Mempool, error) { return c.b.CreateMempool() } -func (c *blockChainWithMetrics) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { - return c.b.InitializeMempool(addrDescForOutpoint) +func (c *blockChainWithMetrics) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc, onNewTxAddr bchain.OnNewTxAddrFunc) error { + return c.b.InitializeMempool(addrDescForOutpoint, onNewTxAddr) } func (c *blockChainWithMetrics) Shutdown(ctx context.Context) error { @@ -293,9 +293,9 @@ func (c *mempoolWithMetrics) observeRPCLatency(method string, start time.Time, e c.m.RPCLatency.With(common.Labels{"method": method, "error": e}).Observe(float64(time.Since(start)) / 1e6) // in milliseconds } -func (c *mempoolWithMetrics) Resync(onNewTxAddr bchain.OnNewTxAddrFunc) (count int, err error) { +func (c *mempoolWithMetrics) Resync() (count int, err error) { defer func(s time.Time) { c.observeRPCLatency("ResyncMempool", s, err) }(time.Now()) - count, err = c.mempool.Resync(onNewTxAddr) + count, err = c.mempool.Resync() if err == nil { c.m.MempoolSize.Set(float64(count)) } diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index a6c4cdf0..b3e17b75 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -139,11 +139,12 @@ func (b *BitcoinRPC) CreateMempool() (bchain.Mempool, error) { } // InitializeMempool creates ZeroMQ subscription and sets AddrDescForOutpointFunc to the Mempool -func (b *BitcoinRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { +func (b *BitcoinRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc, onNewTxAddr bchain.OnNewTxAddrFunc) error { if b.Mempool == nil { return errors.New("Mempool not created") } b.Mempool.AddrDescForOutpoint = addrDescForOutpoint + b.Mempool.OnNewTxAddr = onNewTxAddr if b.mq == nil { mq, err := bchain.NewMQ(b.ChainConfig.MessageQueueBinding, b.pushHandler) if err != nil { diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 18643370..415174a4 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -172,10 +172,11 @@ func (b *EthereumRPC) CreateMempool() (bchain.Mempool, error) { } // InitializeMempool creates subscriptions to newHeads and newPendingTransactions -func (b *EthereumRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { +func (b *EthereumRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc, onNewTxAddr bchain.OnNewTxAddrFunc) error { if b.Mempool == nil { return errors.New("Mempool not created") } + b.Mempool.OnNewTxAddr = onNewTxAddr if b.isETC { glog.Info(b.ChainConfig.CoinName, " does not support subscription to newHeads") } else { diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index ae13459e..887ecee1 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -12,6 +12,11 @@ type addrIndex struct { n int32 } +type txEntry struct { + addrIndexes []addrIndex + time uint32 +} + type txidio struct { txid string io []addrIndex @@ -21,11 +26,11 @@ type txidio struct { type MempoolBitcoinType struct { chain BlockChain mux sync.Mutex - txToInputOutput map[string][]addrIndex + txEntries map[string]txEntry addrDescToTx map[string][]Outpoint chanTxid chan string chanAddrIndex chan txidio - onNewTxAddr OnNewTxAddrFunc + OnNewTxAddr OnNewTxAddrFunc AddrDescForOutpoint AddrDescForOutpointFunc } @@ -79,10 +84,10 @@ func (m *MempoolBitcoinType) GetAddrDescTransactions(addrDesc AddressDescriptor) return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil } -func (m *MempoolBitcoinType) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]Outpoint) { +func (m *MempoolBitcoinType) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { m.mux.Lock() defer m.mux.Unlock() - m.txToInputOutput = newTxToInputOutput + m.txEntries = newTxEntries m.addrDescToTx = newAddrDescToTx } @@ -128,8 +133,8 @@ func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan Outpoint, ch if len(addrDesc) > 0 { io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } - if m.onNewTxAddr != nil { - m.onNewTxAddr(tx, addrDesc) + if m.OnNewTxAddr != nil { + m.OnNewTxAddr(tx, addrDesc) } } dispatched := 0 @@ -166,37 +171,37 @@ func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan Outpoint, ch // Resync gets mempool transactions and maps outputs to transactions. // Resync is not reentrant, it should be called from a single thread. // Read operations (GetTransactions) are safe. -func (m *MempoolBitcoinType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { +func (m *MempoolBitcoinType) Resync() (int, error) { start := time.Now() glog.V(1).Info("mempool: resync") - m.onNewTxAddr = onNewTxAddr txs, err := m.chain.GetMempoolTransactions() if err != nil { return 0, err } glog.V(2).Info("mempool: resync ", len(txs), " txs") // allocate slightly larger capacity of the maps - newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) + newTxEntries := make(map[string]txEntry, len(m.txEntries)+5) newAddrDescToTx := make(map[string][]Outpoint, len(m.addrDescToTx)+5) dispatched := 0 - onNewData := func(txid string, io []addrIndex) { - if len(io) > 0 { - newTxToInputOutput[txid] = io - for _, si := range io { + txTime := uint32(time.Now().Unix()) + onNewData := func(txid string, entry txEntry) { + if len(entry.addrIndexes) > 0 { + newTxEntries[txid] = entry + for _, si := range entry.addrIndexes { newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], Outpoint{txid, si.n}) } } } // get transaction in parallel using goroutines created in NewUTXOMempool for _, txid := range txs { - io, exists := m.txToInputOutput[txid] + io, exists := m.txEntries[txid] if !exists { loop: for { select { // store as many processed transactions as possible case tio := <-m.chanAddrIndex: - onNewData(tio.txid, tio.io) + onNewData(tio.txid, txEntry{tio.io, txTime}) dispatched-- // send transaction to be processed case m.chanTxid <- txid: @@ -210,10 +215,9 @@ func (m *MempoolBitcoinType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { } for i := 0; i < dispatched; i++ { tio := <-m.chanAddrIndex - onNewData(tio.txid, tio.io) + onNewData(tio.txid, txEntry{tio.io, txTime}) } - m.updateMappings(newTxToInputOutput, newAddrDescToTx) - m.onNewTxAddr = nil - glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txToInputOutput), " transactions in mempool") - return len(m.txToInputOutput), nil + m.updateMappings(newTxEntries, newAddrDescToTx) + glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") + return len(m.txEntries), nil } diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index 1f05c9c5..c6033de6 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -9,10 +9,11 @@ import ( // MempoolEthereumType is mempool handle of EthereumType chains type MempoolEthereumType struct { - chain BlockChain - mux sync.Mutex - txToInputOutput map[string][]addrIndex - addrDescToTx map[string][]Outpoint + chain BlockChain + mux sync.Mutex + txEntries map[string]txEntry + addrDescToTx map[string][]Outpoint + OnNewTxAddr OnNewTxAddrFunc } // NewMempoolEthereumType creates new mempool handler. @@ -37,10 +38,10 @@ func (m *MempoolEthereumType) GetAddrDescTransactions(addrDesc AddressDescriptor return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil } -func (m *MempoolEthereumType) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]Outpoint) { +func (m *MempoolEthereumType) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { m.mux.Lock() defer m.mux.Unlock() - m.txToInputOutput = newTxToInputOutput + m.txEntries = newTxEntries m.addrDescToTx = newAddrDescToTx } @@ -56,73 +57,83 @@ func appendAddress(io []addrIndex, i int32, a string, parser BlockChainParser) [ return io } +func (m *MempoolEthereumType) createTxEntry(txid string, txTime uint32) (txEntry, bool) { + tx, err := m.chain.GetTransactionForMempool(txid) + if err != nil { + if err != ErrTxNotFound { + glog.Warning("cannot get transaction ", txid, ": ", err) + } + return txEntry{}, false + } + parser := m.chain.GetChainParser() + addrIndexes := make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) + for _, output := range tx.Vout { + addrDesc, err := parser.GetAddrDescFromVout(&output) + if err != nil { + if err != ErrAddressMissing { + glog.Error("error in output addrDesc in ", txid, " ", output.N, ": ", err) + } + continue + } + if len(addrDesc) > 0 { + addrIndexes = append(addrIndexes, addrIndex{string(addrDesc), int32(output.N)}) + } + } + for _, input := range tx.Vin { + for i, a := range input.Addresses { + addrIndexes = appendAddress(addrIndexes, ^int32(i), a, parser) + } + } + t, err := parser.EthereumTypeGetErc20FromTx(tx) + if err != nil { + glog.Error("GetErc20FromTx for tx ", txid, ", ", err) + } else { + for i := range t { + addrIndexes = appendAddress(addrIndexes, ^int32(i+1), t[i].From, parser) + addrIndexes = appendAddress(addrIndexes, int32(i+1), t[i].To, parser) + } + } + if m.OnNewTxAddr != nil { + sent := make(map[string]struct{}) + for _, si := range addrIndexes { + if _, found := sent[si.addrDesc]; !found { + m.OnNewTxAddr(tx, AddressDescriptor(si.addrDesc)) + sent[si.addrDesc] = struct{}{} + } + } + } + return txEntry{addrIndexes: addrIndexes, time: txTime}, true +} + // Resync gets mempool transactions and maps outputs to transactions. // Resync is not reentrant, it should be called from a single thread. // Read operations (GetTransactions) are safe. -func (m *MempoolEthereumType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { +func (m *MempoolEthereumType) Resync() (int, error) { start := time.Now() glog.V(1).Info("Mempool: resync") txs, err := m.chain.GetMempoolTransactions() if err != nil { return 0, err } - parser := m.chain.GetChainParser() // allocate slightly larger capacity of the maps - newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) + newTxEntries := make(map[string]txEntry, len(m.txEntries)+5) newAddrDescToTx := make(map[string][]Outpoint, len(m.addrDescToTx)+5) + txTime := uint32(time.Now().Unix()) + var ok bool for _, txid := range txs { - io, exists := m.txToInputOutput[txid] + entry, exists := m.txEntries[txid] if !exists { - tx, err := m.chain.GetTransactionForMempool(txid) - if err != nil { - if err != ErrTxNotFound { - glog.Warning("cannot get transaction ", txid, ": ", err) - } + entry, ok = m.createTxEntry(txid, txTime) + if !ok { continue } - io = make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) - for _, output := range tx.Vout { - addrDesc, err := parser.GetAddrDescFromVout(&output) - if err != nil { - if err != ErrAddressMissing { - glog.Error("error in output addrDesc in ", txid, " ", output.N, ": ", err) - } - continue - } - if len(addrDesc) > 0 { - io = append(io, addrIndex{string(addrDesc), int32(output.N)}) - } - } - for _, input := range tx.Vin { - for i, a := range input.Addresses { - appendAddress(io, ^int32(i), a, parser) - } - } - t, err := parser.EthereumTypeGetErc20FromTx(tx) - if err != nil { - glog.Error("GetErc20FromTx for tx ", txid, ", ", err) - } else { - for i := range t { - io = appendAddress(io, ^int32(i+1), t[i].From, parser) - io = appendAddress(io, int32(i+1), t[i].To, parser) - } - } - if onNewTxAddr != nil { - sent := make(map[string]struct{}) - for _, si := range io { - if _, found := sent[si.addrDesc]; !found { - onNewTxAddr(tx, AddressDescriptor(si.addrDesc)) - sent[si.addrDesc] = struct{}{} - } - } - } } - newTxToInputOutput[txid] = io - for _, si := range io { + newTxEntries[txid] = entry + for _, si := range entry.addrIndexes { newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], Outpoint{txid, si.n}) } } - m.updateMappings(newTxToInputOutput, newAddrDescToTx) - glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txToInputOutput), " transactions in mempool") - return len(m.txToInputOutput), nil + m.updateMappings(newTxEntries, newAddrDescToTx) + glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") + return len(m.txEntries), nil } diff --git a/bchain/types.go b/bchain/types.go index 0415f4b8..368965fa 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -202,7 +202,7 @@ type BlockChain interface { // create mempool but do not initialize it CreateMempool() (Mempool, error) // initialize mempool, create ZeroMQ (or other) subscription - InitializeMempool(AddrDescForOutpointFunc) error + InitializeMempool(AddrDescForOutpointFunc, OnNewTxAddrFunc) error // shutdown mempool, ZeroMQ and block chain connections Shutdown(ctx context.Context) error // chain info @@ -278,7 +278,7 @@ type BlockChainParser interface { // Mempool defines common interface to mempool type Mempool interface { - Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) + Resync() (int, error) GetTransactions(address string) ([]Outpoint, error) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) } diff --git a/blockbook.go b/blockbook.go index e694bcc2..4630b263 100644 --- a/blockbook.go +++ b/blockbook.go @@ -231,13 +231,13 @@ func main() { if chain.GetChainParser().GetChainType() == bchain.ChainBitcoinType { addrDescForOutpoint = index.AddrDescForOutpoint } - err = chain.InitializeMempool(addrDescForOutpoint) + err = chain.InitializeMempool(addrDescForOutpoint, onNewTxAddr) if err != nil { glog.Error("initializeMempool ", err) return } var mempoolCount int - if mempoolCount, err = mempool.Resync(nil); err != nil { + if mempoolCount, err = mempool.Resync(); err != nil { glog.Error("resyncMempool ", err) return } @@ -250,6 +250,8 @@ func main() { if publicServer != nil { // start full public interface + callbacksOnNewBlock = append(callbacksOnNewBlock, publicServer.OnNewBlock) + callbacksOnNewTxAddr = append(callbacksOnNewTxAddr, publicServer.OnNewTxAddr) publicServer.ConnectFullPublicInterface() } @@ -342,8 +344,6 @@ func startPublicServer() (*server.PublicServer, error) { } } }() - callbacksOnNewBlock = append(callbacksOnNewBlock, publicServer.OnNewBlock) - callbacksOnNewTxAddr = append(callbacksOnNewTxAddr, publicServer.OnNewTxAddr) return publicServer, err } @@ -474,7 +474,7 @@ func syncMempoolLoop() { // resync mempool about every minute if there are no chanSyncMempool requests, with debounce 1 second tickAndDebounce(time.Duration(*resyncMempoolPeriodMs)*time.Millisecond, debounceResyncMempoolMs*time.Millisecond, chanSyncMempool, func() { internalState.StartedMempoolSync() - if count, err := mempool.Resync(onNewTxAddr); err != nil { + if count, err := mempool.Resync(); err != nil { glog.Error("syncMempoolLoop ", errors.ErrorStack(err)) } else { internalState.FinishedMempoolSync(count) diff --git a/tests/dbtestdata/fakechain.go b/tests/dbtestdata/fakechain.go index 270403bb..d64968c7 100644 --- a/tests/dbtestdata/fakechain.go +++ b/tests/dbtestdata/fakechain.go @@ -25,7 +25,7 @@ func (c *fakeBlockChain) Initialize() error { return nil } -func (c *fakeBlockChain) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc) error { +func (c *fakeBlockChain) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc, onNewTxAddr bchain.OnNewTxAddrFunc) error { return nil } diff --git a/tests/integration.go b/tests/integration.go index d1e70130..6d40b730 100644 --- a/tests/integration.go +++ b/tests/integration.go @@ -174,7 +174,7 @@ func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, bc return nil, nil, fmt.Errorf("Mempool creation failed: %s", err) } - err = cli.InitializeMempool(nil) + err = cli.InitializeMempool(nil, nil) if err != nil { return nil, nil, fmt.Errorf("Mempool initialization failed: %s", err) } diff --git a/tests/rpc/rpc.go b/tests/rpc/rpc.go index c4122d3f..d33e6fb7 100644 --- a/tests/rpc/rpc.go +++ b/tests/rpc/rpc.go @@ -200,7 +200,7 @@ func testMempoolSync(t *testing.T, h *TestHandler) { for i := 0; i < 3; i++ { txs := getMempool(t, h) - n, err := h.Mempool.Resync(nil) + n, err := h.Mempool.Resync() if err != nil { t.Fatal(err) } From 987aec47f945e5012373b922c8e77fe9aef57e97 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 2 Apr 2019 11:39:38 +0200 Subject: [PATCH 16/39] Show mempool content in explorer --- api/types.go | 13 +++++++++++++ api/worker.go | 23 +++++++++++++++++++++++ bchain/coins/blockchain.go | 5 +++++ bchain/mempool_bitcoin_type.go | 33 +++++++++++++++++++++++++++++++++ bchain/mempool_ethereum_type.go | 5 +++++ bchain/types.go | 10 ++++++++++ server/public.go | 24 ++++++++++++++++++++++++ static/css/main.css | 5 +++++ static/templates/index.html | 4 ++++ static/templates/mempool.html | 27 +++++++++++++++++++++++++++ 10 files changed, 149 insertions(+) create mode 100644 static/templates/mempool.html diff --git a/api/types.go b/api/types.go index a2ddde4a..e0e562ca 100644 --- a/api/types.go +++ b/api/types.go @@ -337,3 +337,16 @@ type SystemInfo struct { Blockbook *BlockbookInfo `json:"blockbook"` Backend *bchain.ChainInfo `json:"backend"` } + +// MempoolTxid contains information about a transaction in mempool +type MempoolTxid struct { + Time int64 `json:"time"` + Txid string `json:"txid"` +} + +// MempoolTxids contains a list of mempool txids with paging information +type MempoolTxids struct { + Paging + Mempool []MempoolTxid `json:"mempool"` + MempoolSize int `json:"mempoolSize"` +} diff --git a/api/worker.go b/api/worker.go index 806bcf2c..b19b1e11 100644 --- a/api/worker.go +++ b/api/worker.go @@ -1070,3 +1070,26 @@ func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) { glog.Info("GetSystemInfo finished in ", time.Since(start)) return &SystemInfo{bi, ci}, nil } + +// GetMempool returns a page of mempool txids +func (w *Worker) GetMempool(page int, itemsOnPage int) (*MempoolTxids, error) { + page-- + if page < 0 { + page = 0 + } + entries := w.mempool.GetAllEntries() + pg, from, to, page := computePaging(len(entries), page, itemsOnPage) + r := &MempoolTxids{ + Paging: pg, + MempoolSize: len(entries), + } + r.Mempool = make([]MempoolTxid, to-from) + for i := from; i < to; i++ { + entry := &entries[i] + r.Mempool[i-from] = MempoolTxid{ + Txid: entry.Txid, + Time: int64(entry.Time), + } + } + return r, nil +} diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 6f1913fc..2567c77f 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -311,3 +311,8 @@ func (c *mempoolWithMetrics) GetAddrDescTransactions(addrDesc bchain.AddressDesc defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactionsForAddrDesc", s, err) }(time.Now()) return c.mempool.GetAddrDescTransactions(addrDesc) } + +func (c *mempoolWithMetrics) GetAllEntries() (v bchain.MempoolTxidEntries) { + defer func(s time.Time) { c.observeRPCLatency("GetAllEntries", s, nil) }(time.Now()) + return c.mempool.GetAllEntries() +} diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index 887ecee1..8dba0e5b 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -1,6 +1,7 @@ package bchain import ( + "sort" "sync" "time" @@ -221,3 +222,35 @@ func (m *MempoolBitcoinType) Resync() (int, error) { glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") return len(m.txEntries), nil } + +func (a MempoolTxidEntries) Len() int { return len(a) } +func (a MempoolTxidEntries) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a MempoolTxidEntries) Less(i, j int) bool { + // if the Time is equal, sort by txid to make the order defined + hi := a[i].Time + hj := a[j].Time + if hi == hj { + return a[i].Txid > a[j].Txid + } + // order in reverse + return hi > hj +} + +func getAllEntries(txEntries map[string]txEntry) MempoolTxidEntries { + a := make(MempoolTxidEntries, len(txEntries)) + i := 0 + for txid, entry := range txEntries { + a[i] = MempoolTxidEntry{ + Txid: txid, + Time: entry.time, + } + i++ + } + sort.Sort(a) + return a +} + +// GetAllEntries returns all mempool entries sorted by fist seen time in descending order +func (m *MempoolBitcoinType) GetAllEntries() MempoolTxidEntries { + return getAllEntries(m.txEntries) +} diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index c6033de6..e81744d3 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -137,3 +137,8 @@ func (m *MempoolEthereumType) Resync() (int, error) { glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") return len(m.txEntries), nil } + +// GetAllEntries returns all mempool entries sorted by fist seen time in descending order +func (m *MempoolEthereumType) GetAllEntries() MempoolTxidEntries { + return getAllEntries(m.txEntries) +} diff --git a/bchain/types.go b/bchain/types.go index 368965fa..58b1acf9 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -185,6 +185,15 @@ type Erc20Transfer struct { Tokens big.Int } +// MempoolTxidEntry contains mempool txid with first seen time +type MempoolTxidEntry struct { + Txid string + Time uint32 +} + +// MempoolTxidEntries is array of MempoolTxidEntry +type MempoolTxidEntries []MempoolTxidEntry + // OnNewBlockFunc is used to send notification about a new block type OnNewBlockFunc func(hash string, height uint32) @@ -281,4 +290,5 @@ type Mempool interface { Resync() (int, error) GetTransactions(address string) ([]Outpoint, error) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) + GetAllEntries() MempoolTxidEntries } diff --git a/server/public.go b/server/public.go index 7543ec2a..60ca2e53 100644 --- a/server/public.go +++ b/server/public.go @@ -26,6 +26,7 @@ import ( const txsOnPage = 25 const blocksOnPage = 50 +const mempoolTxsOnPage = 50 const txsInAPI = 1000 const ( @@ -139,6 +140,7 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"block/", s.htmlTemplateHandler(s.explorerBlock)) serveMux.HandleFunc(path+"spending/", s.htmlTemplateHandler(s.explorerSpendingTx)) serveMux.HandleFunc(path+"sendtx", s.htmlTemplateHandler(s.explorerSendTx)) + serveMux.HandleFunc(path+"mempool", s.htmlTemplateHandler(s.explorerMempool)) } else { // redirect to wallet requests for tx and address, possibly to external site serveMux.HandleFunc(path+"tx/", s.txRedirect) @@ -384,6 +386,7 @@ const ( blocksTpl blockTpl sendTransactionTpl + mempoolTpl tplCount ) @@ -402,6 +405,7 @@ type TemplateData struct { Blocks *api.Blocks Block *api.Block Info *api.SystemInfo + MempoolTxids *api.MempoolTxids Page int PrevPage int NextPage int @@ -477,6 +481,7 @@ func (s *PublicServer) parseTemplates() []*template.Template { t[blockTpl] = createTemplate("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") } t[xpubTpl] = createTemplate("./static/templates/xpub.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html") + t[mempoolTpl] = createTemplate("./static/templates/mempool.html", "./static/templates/paging.html", "./static/templates/base.html") return t } @@ -798,6 +803,25 @@ func (s *PublicServer) explorerSendTx(w http.ResponseWriter, r *http.Request) (t return sendTransactionTpl, data, nil } +func (s *PublicServer) explorerMempool(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { + var mempoolTxids *api.MempoolTxids + var err error + s.metrics.ExplorerViews.With(common.Labels{"action": "mempool"}).Inc() + page, ec := strconv.Atoi(r.URL.Query().Get("page")) + if ec != nil { + page = 0 + } + mempoolTxids, err = s.api.GetMempool(page, mempoolTxsOnPage) + if err != nil { + return errorTpl, nil, err + } + data := s.newTemplateData() + data.MempoolTxids = mempoolTxids + data.Page = mempoolTxids.Page + data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(mempoolTxids.Page, mempoolTxids.TotalPages) + return mempoolTpl, data, nil +} + func getPagingRange(page int, total int) ([]int, int, int) { // total==-1 means total is unknown, show only prev/next buttons if total >= 0 && total < 2 { diff --git a/static/css/main.css b/static/css/main.css index d54f9841..c0bf1ee2 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -268,6 +268,11 @@ table.data-table table.data-table th { margin: 0; } +.h-container h5 { + margin-top: 6px; + margin-bottom: 0; +} + .page-link { color: #428bca; } diff --git a/static/templates/index.html b/static/templates/index.html index f6638adc..5e275fc3 100644 --- a/static/templates/index.html +++ b/static/templates/index.html @@ -43,6 +43,10 @@ Last Mempool Update {{formatTime $bb.LastMempoolTime}} + + Transactions in Mempool + {{if .InternalExplorer}}{{$bb.MempoolSize}}{{else}}{{$bb.MempoolSize}}{{end}} + Size On Disk {{$bb.DbSize}} diff --git a/static/templates/mempool.html b/static/templates/mempool.html new file mode 100644 index 00000000..eb8b0da8 --- /dev/null +++ b/static/templates/mempool.html @@ -0,0 +1,27 @@ +{{define "specific"}}{{$txs := .MempoolTxids.Mempool}}{{$data := .}} +

Mempool Transactions by first seen time +

+
+
{{$.MempoolTxids.MempoolSize}} Transactions in mempool
+ +
+
+ + + + + + + + + {{- range $tx := $txs -}} + + + + + {{- end -}} + +
TransactionFirst Seen Time
{{$tx.Txid}}{{formatUnixTime $tx.Time}}
+
+ +{{end}} \ No newline at end of file From c19f6bfb424f9c2d91bd72e0b058f3cc5dd10ae8 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 3 Apr 2019 12:51:51 +0200 Subject: [PATCH 17/39] Fix tryParseOmni --- bchain/coins/btc/bitcoinparser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bchain/coins/btc/bitcoinparser.go b/bchain/coins/btc/bitcoinparser.go index eed0878c..c1b6a212 100644 --- a/bchain/coins/btc/bitcoinparser.go +++ b/bchain/coins/btc/bitcoinparser.go @@ -160,7 +160,7 @@ func (p *BitcoinParser) tryParseOmni(data []byte) string { // currently only simple send transaction version 0 is supported, see // https://github.com/OmniLayer/spec#transfer-coins-simple-send - if len(data) != 20 && data[0] != 'o' { + if len(data) != 20 || data[0] != 'o' { return "" } // omni (4) (2) (2) From 870354bc90e9734feb538fa0e23ae04cb8315890 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Tue, 2 Apr 2019 16:41:47 +0200 Subject: [PATCH 18/39] Extract mempool common functionality to BaseMempool --- bchain/basemempool.go | 86 +++++++++++++++++++++++++++++++++ bchain/mempool_bitcoin_type.go | 83 ++----------------------------- bchain/mempool_ethereum_type.go | 38 +-------------- 3 files changed, 92 insertions(+), 115 deletions(-) create mode 100644 bchain/basemempool.go diff --git a/bchain/basemempool.go b/bchain/basemempool.go new file mode 100644 index 00000000..94d85fa9 --- /dev/null +++ b/bchain/basemempool.go @@ -0,0 +1,86 @@ +package bchain + +import ( + "sort" + "sync" +) + +type addrIndex struct { + addrDesc string + n int32 +} + +type txEntry struct { + addrIndexes []addrIndex + time uint32 +} + +type txidio struct { + txid string + io []addrIndex +} + +// BaseMempool is mempool base handle +type BaseMempool struct { + chain BlockChain + mux sync.Mutex + txEntries map[string]txEntry + addrDescToTx map[string][]Outpoint + OnNewTxAddr OnNewTxAddrFunc +} + +// GetTransactions returns slice of mempool transactions for given address +func (m *BaseMempool) GetTransactions(address string) ([]Outpoint, error) { + parser := m.chain.GetChainParser() + addrDesc, err := parser.GetAddrDescFromAddress(address) + if err != nil { + return nil, err + } + return m.GetAddrDescTransactions(addrDesc) +} + +// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor +func (m *BaseMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) { + m.mux.Lock() + defer m.mux.Unlock() + return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil +} + +func (a MempoolTxidEntries) Len() int { return len(a) } +func (a MempoolTxidEntries) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a MempoolTxidEntries) Less(i, j int) bool { + // if the Time is equal, sort by txid to make the order defined + hi := a[i].Time + hj := a[j].Time + if hi == hj { + return a[i].Txid > a[j].Txid + } + // order in reverse + return hi > hj +} + +func (m *BaseMempool) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { + m.mux.Lock() + defer m.mux.Unlock() + m.txEntries = newTxEntries + m.addrDescToTx = newAddrDescToTx +} + +func getAllEntries(txEntries map[string]txEntry) MempoolTxidEntries { + a := make(MempoolTxidEntries, len(txEntries)) + i := 0 + for txid, entry := range txEntries { + a[i] = MempoolTxidEntry{ + Txid: txid, + Time: entry.time, + } + i++ + } + sort.Sort(a) + return a +} + +// GetAllEntries returns all mempool entries sorted by fist seen time in descending order +func (m *BaseMempool) GetAllEntries() MempoolTxidEntries { + return getAllEntries(m.txEntries) +} diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index 8dba0e5b..929dd26c 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -1,37 +1,16 @@ package bchain import ( - "sort" - "sync" "time" "github.com/golang/glog" ) -type addrIndex struct { - addrDesc string - n int32 -} - -type txEntry struct { - addrIndexes []addrIndex - time uint32 -} - -type txidio struct { - txid string - io []addrIndex -} - // MempoolBitcoinType is mempool handle. type MempoolBitcoinType struct { - chain BlockChain - mux sync.Mutex - txEntries map[string]txEntry - addrDescToTx map[string][]Outpoint + BaseMempool chanTxid chan string chanAddrIndex chan txidio - OnNewTxAddr OnNewTxAddrFunc AddrDescForOutpoint AddrDescForOutpointFunc } @@ -39,7 +18,9 @@ type MempoolBitcoinType struct { // For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *MempoolBitcoinType { m := &MempoolBitcoinType{ - chain: chain, + BaseMempool: BaseMempool{ + chain: chain, + }, chanTxid: make(chan string, 1), chanAddrIndex: make(chan txidio, 1), } @@ -68,30 +49,6 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *Mempo return m } -// GetTransactions returns slice of mempool transactions for given address -func (m *MempoolBitcoinType) GetTransactions(address string) ([]Outpoint, error) { - parser := m.chain.GetChainParser() - addrDesc, err := parser.GetAddrDescFromAddress(address) - if err != nil { - return nil, err - } - return m.GetAddrDescTransactions(addrDesc) -} - -// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor -func (m *MempoolBitcoinType) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) { - m.mux.Lock() - defer m.mux.Unlock() - return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil -} - -func (m *MempoolBitcoinType) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { - m.mux.Lock() - defer m.mux.Unlock() - m.txEntries = newTxEntries - m.addrDescToTx = newAddrDescToTx -} - func (m *MempoolBitcoinType) getInputAddress(input Outpoint) *addrIndex { var addrDesc AddressDescriptor if m.AddrDescForOutpoint != nil { @@ -222,35 +179,3 @@ func (m *MempoolBitcoinType) Resync() (int, error) { glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") return len(m.txEntries), nil } - -func (a MempoolTxidEntries) Len() int { return len(a) } -func (a MempoolTxidEntries) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a MempoolTxidEntries) Less(i, j int) bool { - // if the Time is equal, sort by txid to make the order defined - hi := a[i].Time - hj := a[j].Time - if hi == hj { - return a[i].Txid > a[j].Txid - } - // order in reverse - return hi > hj -} - -func getAllEntries(txEntries map[string]txEntry) MempoolTxidEntries { - a := make(MempoolTxidEntries, len(txEntries)) - i := 0 - for txid, entry := range txEntries { - a[i] = MempoolTxidEntry{ - Txid: txid, - Time: entry.time, - } - i++ - } - sort.Sort(a) - return a -} - -// GetAllEntries returns all mempool entries sorted by fist seen time in descending order -func (m *MempoolBitcoinType) GetAllEntries() MempoolTxidEntries { - return getAllEntries(m.txEntries) -} diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index e81744d3..315f166a 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -1,7 +1,6 @@ package bchain import ( - "sync" "time" "github.com/golang/glog" @@ -9,40 +8,12 @@ import ( // MempoolEthereumType is mempool handle of EthereumType chains type MempoolEthereumType struct { - chain BlockChain - mux sync.Mutex - txEntries map[string]txEntry - addrDescToTx map[string][]Outpoint - OnNewTxAddr OnNewTxAddrFunc + BaseMempool } // NewMempoolEthereumType creates new mempool handler. func NewMempoolEthereumType(chain BlockChain) *MempoolEthereumType { - return &MempoolEthereumType{chain: chain} -} - -// GetTransactions returns slice of mempool transactions for given address -func (m *MempoolEthereumType) GetTransactions(address string) ([]Outpoint, error) { - parser := m.chain.GetChainParser() - addrDesc, err := parser.GetAddrDescFromAddress(address) - if err != nil { - return nil, err - } - return m.GetAddrDescTransactions(addrDesc) -} - -// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor -func (m *MempoolEthereumType) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) { - m.mux.Lock() - defer m.mux.Unlock() - return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil -} - -func (m *MempoolEthereumType) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { - m.mux.Lock() - defer m.mux.Unlock() - m.txEntries = newTxEntries - m.addrDescToTx = newAddrDescToTx + return &MempoolEthereumType{BaseMempool: BaseMempool{chain: chain}} } func appendAddress(io []addrIndex, i int32, a string, parser BlockChainParser) []addrIndex { @@ -137,8 +108,3 @@ func (m *MempoolEthereumType) Resync() (int, error) { glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") return len(m.txEntries), nil } - -// GetAllEntries returns all mempool entries sorted by fist seen time in descending order -func (m *MempoolEthereumType) GetAllEntries() MempoolTxidEntries { - return getAllEntries(m.txEntries) -} From 47f798dbaa15a07e239eb062d9c285940dafcf6b Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 3 Apr 2019 14:02:15 +0200 Subject: [PATCH 19/39] Return mempool transactions for address in reverse order --- bchain/basemempool.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/bchain/basemempool.go b/bchain/basemempool.go index 94d85fa9..01e1b90b 100644 --- a/bchain/basemempool.go +++ b/bchain/basemempool.go @@ -39,11 +39,17 @@ func (m *BaseMempool) GetTransactions(address string) ([]Outpoint, error) { return m.GetAddrDescTransactions(addrDesc) } -// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor +// GetAddrDescTransactions returns slice of mempool transactions for given address descriptor, in reverse order func (m *BaseMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) { m.mux.Lock() defer m.mux.Unlock() - return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil + outpoints := m.addrDescToTx[string(addrDesc)] + rv := make([]Outpoint, len(outpoints)) + for i, j := len(outpoints)-1, 0; i >= 0; i-- { + rv[j] = outpoints[i] + j++ + } + return rv, nil } func (a MempoolTxidEntries) Len() int { return len(a) } @@ -84,3 +90,12 @@ func getAllEntries(txEntries map[string]txEntry) MempoolTxidEntries { func (m *BaseMempool) GetAllEntries() MempoolTxidEntries { return getAllEntries(m.txEntries) } + +// GetTransactionTime returns first seen time of a transaction +func (m *BaseMempool) GetTransactionTime(txid string) uint32 { + e, found := m.txEntries[txid] + if !found { + return 0 + } + return e.time +} From b64d76d8f92ceee862a8a795f9b82327ae482cd1 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 3 Apr 2019 14:08:36 +0200 Subject: [PATCH 20/39] Show first seen date of mempool transaction in explorer --- api/worker.go | 4 ++++ api/xpub.go | 26 +++++++++++++-------- bchain/coins/blockchain.go | 4 ++++ bchain/types.go | 1 + static/templates/txdetail.html | 4 +--- static/templates/txdetail_ethereumtype.html | 4 +--- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/api/worker.go b/api/worker.go index b19b1e11..8209c83b 100644 --- a/api/worker.go +++ b/api/worker.go @@ -294,6 +294,10 @@ func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height uint32, return nil, err } } + // for mempool transaction get first seen time + if bchainTx.Confirmations == 0 { + bchainTx.Blocktime = int64(w.mempool.GetTransactionTime(bchainTx.Txid)) + } r := &Tx{ Blockhash: blockhash, Blockheight: int(height), diff --git a/api/xpub.go b/api/xpub.go index 7508f24a..664a12ff 100644 --- a/api/xpub.go +++ b/api/xpub.go @@ -383,13 +383,13 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc return nil, err } // setup filtering of txids - var useTxids func(txid *xpubTxid, ad *xpubAddress) bool + var txidFilter func(txid *xpubTxid, ad *xpubAddress) bool if !(filter.FromHeight == 0 && filter.ToHeight == 0 && filter.Vout == AddressFilterVoutOff) { toHeight := maxUint32 if filter.ToHeight != 0 { toHeight = filter.ToHeight } - useTxids = func(txid *xpubTxid, ad *xpubAddress) bool { + txidFilter = func(txid *xpubTxid, ad *xpubAddress) bool { if txid.height < filter.FromHeight || txid.height > toHeight { return false } @@ -406,6 +406,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc // process mempool, only if ToHeight is not specified if filter.ToHeight == 0 && !filter.OnlyConfirmed { txmMap = make(map[string]*Tx) + mempoolEntries := make(bchain.MempoolTxidEntries, 0) for _, da := range [][]xpubAddress{data.addresses, data.changeAddresses} { for i := range da { ad := &da[i] @@ -432,18 +433,23 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc } uBalSat.Add(&uBalSat, tx.getAddrVoutValue(ad.addrDesc)) uBalSat.Sub(&uBalSat, tx.getAddrVinValue(ad.addrDesc)) - if page == 0 && !foundTx && (useTxids == nil || useTxids(&txid, ad)) { - if option == AccountDetailsTxidHistory { - txids = append(txids, tx.Txid) - } else if option >= AccountDetailsTxHistoryLight { - txs = append(txs, tx) - } + // mempool txs are returned only on the first page, uniquely and filtered + if page == 0 && !foundTx && (txidFilter == nil || txidFilter(&txid, ad)) { + mempoolEntries = append(mempoolEntries, bchain.MempoolTxidEntry{Txid: txid.txid, Time: uint32(tx.Blocktime)}) } } - } } } + // sort the entries by time descending + sort.Sort(mempoolEntries) + for _, entry := range mempoolEntries { + if option == AccountDetailsTxidHistory { + txids = append(txids, entry.Txid) + } else if option >= AccountDetailsTxHistoryLight { + txs = append(txs, txmMap[entry.Txid]) + } + } } if option >= AccountDetailsTxidHistory { txcMap := make(map[string]bool) @@ -459,7 +465,7 @@ func (w *Worker) GetXpubAddress(xpub string, page int, txsOnPage int, option Acc } // add tx only once if !added { - add := useTxids == nil || useTxids(&txid, ad) + add := txidFilter == nil || txidFilter(&txid, ad) txcMap[txid.txid] = add if add { txc = append(txc, txid) diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 2567c77f..38224311 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -316,3 +316,7 @@ func (c *mempoolWithMetrics) GetAllEntries() (v bchain.MempoolTxidEntries) { defer func(s time.Time) { c.observeRPCLatency("GetAllEntries", s, nil) }(time.Now()) return c.mempool.GetAllEntries() } + +func (c *mempoolWithMetrics) GetTransactionTime(txid string) uint32 { + return c.mempool.GetTransactionTime(txid) +} diff --git a/bchain/types.go b/bchain/types.go index 58b1acf9..4c072574 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -291,4 +291,5 @@ type Mempool interface { GetTransactions(address string) ([]Outpoint, error) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) GetAllEntries() MempoolTxidEntries + GetTransactionTime(txid string) uint32 } diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 6b3a3f82..6e7327c8 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -4,9 +4,7 @@
{{$tx.Txid}}
- {{- if $tx.Confirmations -}} -
mined {{formatUnixTime $tx.Blocktime}}
- {{- end -}} +
{{if $tx.Confirmations}}mined{{else}}first seen{{end}} {{formatUnixTime $tx.Blocktime}}
diff --git a/static/templates/txdetail_ethereumtype.html b/static/templates/txdetail_ethereumtype.html index bb71a8cc..78cde7bd 100644 --- a/static/templates/txdetail_ethereumtype.html +++ b/static/templates/txdetail_ethereumtype.html @@ -5,9 +5,7 @@ {{$tx.Txid}} {{if eq $tx.EthereumSpecific.Status 1}} ✔{{end}}{{if eq $tx.EthereumSpecific.Status 0}} ✘{{end}}
- {{- if $tx.Confirmations -}} -
mined {{formatUnixTime $tx.Blocktime}}
- {{- end -}} +
{{if $tx.Confirmations}}mined{{else}}first seen{{end}} {{formatUnixTime $tx.Blocktime}}
From 4512a5713446e82acd8f5c32c9e0c34dd0145f57 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 3 Apr 2019 14:09:16 +0200 Subject: [PATCH 21/39] Bump blockbook to v0.2.2 --- configs/environ.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/environ.json b/configs/environ.json index 6171a41b..f57fa21d 100644 --- a/configs/environ.json +++ b/configs/environ.json @@ -1,5 +1,5 @@ { - "version": "0.2.1", + "version": "0.2.2", "backend_install_path": "/opt/coins/nodes", "backend_data_path": "/opt/coins/data", "blockbook_install_path": "/opt/coins/blockbook", From 4435dbbfb4da3a7ede3fed649018f04eb157f95f Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 3 Apr 2019 22:08:32 +0200 Subject: [PATCH 22/39] Pass correct blockchain object to mempool --- bchain/coins/blockchain.go | 6 +++--- bchain/coins/btc/bitcoinrpc.go | 4 ++-- bchain/coins/eth/ethrpc.go | 4 ++-- bchain/types.go | 2 +- server/public_test.go | 2 +- tests/dbtestdata/fakechain.go | 4 ++-- tests/integration.go | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 38224311..7b39eae4 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -124,7 +124,7 @@ func NewBlockChain(coin string, configfile string, pushHandler func(bchain.Notif if err != nil { return nil, nil, err } - mempool, err := bc.CreateMempool() + mempool, err := bc.CreateMempool(bc) if err != nil { return nil, nil, err } @@ -148,8 +148,8 @@ func (c *blockChainWithMetrics) Initialize() error { return c.b.Initialize() } -func (c *blockChainWithMetrics) CreateMempool() (bchain.Mempool, error) { - return c.b.CreateMempool() +func (c *blockChainWithMetrics) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) { + return c.b.CreateMempool(chain) } func (c *blockChainWithMetrics) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOutpointFunc, onNewTxAddr bchain.OnNewTxAddrFunc) error { diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index b3e17b75..ffda6650 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -131,9 +131,9 @@ func (b *BitcoinRPC) Initialize() error { } // CreateMempool creates mempool if not already created, however does not initialize it -func (b *BitcoinRPC) CreateMempool() (bchain.Mempool, error) { +func (b *BitcoinRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) { if b.Mempool == nil { - b.Mempool = bchain.NewMempoolBitcoinType(b, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers) + b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers) } return b.Mempool, nil } diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 415174a4..5b0d3546 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -164,9 +164,9 @@ func (b *EthereumRPC) Initialize() error { } // CreateMempool creates mempool if not already created, however does not initialize it -func (b *EthereumRPC) CreateMempool() (bchain.Mempool, error) { +func (b *EthereumRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) { if b.Mempool == nil { - b.Mempool = bchain.NewMempoolEthereumType(b) + b.Mempool = bchain.NewMempoolEthereumType(chain) } return b.Mempool, nil } diff --git a/bchain/types.go b/bchain/types.go index 4c072574..4b3c1abd 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -209,7 +209,7 @@ type BlockChain interface { // initialize the block chain connector Initialize() error // create mempool but do not initialize it - CreateMempool() (Mempool, error) + CreateMempool(BlockChain) (Mempool, error) // initialize mempool, create ZeroMQ (or other) subscription InitializeMempool(AddrDescForOutpointFunc, OnNewTxAddrFunc) error // shutdown mempool, ZeroMQ and block chain connections diff --git a/server/public_test.go b/server/public_test.go index aa64044e..ccff606b 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -85,7 +85,7 @@ func setupPublicHTTPServer(t *testing.T) (*PublicServer, string) { glog.Fatal("fakechain: ", err) } - mempool, err := chain.CreateMempool() + mempool, err := chain.CreateMempool(chain) if err != nil { glog.Fatal("mempool: ", err) } diff --git a/tests/dbtestdata/fakechain.go b/tests/dbtestdata/fakechain.go index d64968c7..cee7c16b 100644 --- a/tests/dbtestdata/fakechain.go +++ b/tests/dbtestdata/fakechain.go @@ -17,8 +17,8 @@ func NewFakeBlockChain(parser bchain.BlockChainParser) (bchain.BlockChain, error return &fakeBlockChain{&bchain.BaseChain{Parser: parser}}, nil } -func (b *fakeBlockChain) CreateMempool() (bchain.Mempool, error) { - return bchain.NewMempoolBitcoinType(b, 1, 1), nil +func (b *fakeBlockChain) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) { + return bchain.NewMempoolBitcoinType(chain, 1, 1), nil } func (c *fakeBlockChain) Initialize() error { diff --git a/tests/integration.go b/tests/integration.go index 6d40b730..a33026b3 100644 --- a/tests/integration.go +++ b/tests/integration.go @@ -169,7 +169,7 @@ func initBlockChain(coinName string, cfg json.RawMessage) (bchain.BlockChain, bc return nil, nil, fmt.Errorf("BlockChain initialization failed: %s", err) } - mempool, err := cli.CreateMempool() + mempool, err := cli.CreateMempool(cli) if err != nil { return nil, nil, fmt.Errorf("Mempool creation failed: %s", err) } From 3f973bf47d308c4ac563543ceba97a90c342ef5d Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 4 Apr 2019 23:35:38 +0200 Subject: [PATCH 23/39] Implement new ethereum mempool sync with tx timeout --- bchain/basemempool.go | 24 +++---- bchain/coins/eth/ethrpc.go | 94 ++++++++++++++------------- bchain/mempool_ethereum_type.go | 108 ++++++++++++++++++++++++-------- 3 files changed, 140 insertions(+), 86 deletions(-) diff --git a/bchain/basemempool.go b/bchain/basemempool.go index 01e1b90b..807f1ed6 100644 --- a/bchain/basemempool.go +++ b/bchain/basemempool.go @@ -67,33 +67,33 @@ func (a MempoolTxidEntries) Less(i, j int) bool { func (m *BaseMempool) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { m.mux.Lock() - defer m.mux.Unlock() m.txEntries = newTxEntries m.addrDescToTx = newAddrDescToTx + m.mux.Unlock() } -func getAllEntries(txEntries map[string]txEntry) MempoolTxidEntries { - a := make(MempoolTxidEntries, len(txEntries)) +// GetAllEntries returns all mempool entries sorted by fist seen time in descending order +func (m *BaseMempool) GetAllEntries() MempoolTxidEntries { + entries := make(MempoolTxidEntries, len(m.txEntries)) i := 0 - for txid, entry := range txEntries { - a[i] = MempoolTxidEntry{ + m.mux.Lock() + for txid, entry := range m.txEntries { + entries[i] = MempoolTxidEntry{ Txid: txid, Time: entry.time, } i++ } - sort.Sort(a) - return a -} - -// GetAllEntries returns all mempool entries sorted by fist seen time in descending order -func (m *BaseMempool) GetAllEntries() MempoolTxidEntries { - return getAllEntries(m.txEntries) + m.mux.Unlock() + sort.Sort(entries) + return entries } // GetTransactionTime returns first seen time of a transaction func (m *BaseMempool) GetTransactionTime(txid string) uint32 { + m.mux.Lock() e, found := m.txEntries[txid] + m.mux.Unlock() if !found { return 0 } diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 5b0d3546..9656e6c3 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -41,22 +41,21 @@ type Configuration struct { // EthereumRPC is an interface to JSON-RPC eth service. type EthereumRPC struct { *bchain.BaseChain - client *ethclient.Client - rpc *rpc.Client - timeout time.Duration - Parser *EthereumParser - Mempool *bchain.MempoolEthereumType - bestHeaderLock sync.Mutex - bestHeader *ethtypes.Header - bestHeaderTime time.Time - chanNewBlock chan *ethtypes.Header - newBlockSubscription *rpc.ClientSubscription - chanNewTx chan ethcommon.Hash - newTxSubscription *rpc.ClientSubscription - pendingTransactions map[string]struct{} - pendingTransactionsLock sync.Mutex - ChainConfig *Configuration - isETC bool + client *ethclient.Client + rpc *rpc.Client + timeout time.Duration + Parser *EthereumParser + Mempool *bchain.MempoolEthereumType + mempoolInitialized bool + bestHeaderLock sync.Mutex + bestHeader *ethtypes.Header + bestHeaderTime time.Time + chanNewBlock chan *ethtypes.Header + newBlockSubscription *rpc.ClientSubscription + chanNewTx chan ethcommon.Hash + newTxSubscription *rpc.ClientSubscription + ChainConfig *Configuration + isETC bool } // NewEthereumRPC returns new EthRPC instance. @@ -78,11 +77,10 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification ec := ethclient.NewClient(rc) s := &EthereumRPC{ - BaseChain: &bchain.BaseChain{}, - client: ec, - rpc: rc, - ChainConfig: &c, - pendingTransactions: make(map[string]struct{}), + BaseChain: &bchain.BaseChain{}, + client: ec, + rpc: rc, + ChainConfig: &c, } // always create parser @@ -125,9 +123,7 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification if glog.V(2) { glog.Info("rpc: new tx ", hex) } - s.pendingTransactionsLock.Lock() - s.pendingTransactions[hex] = struct{}{} - s.pendingTransactionsLock.Unlock() + s.Mempool.AddTransactionToMempool(hex) pushHandler(bchain.NotificationNewTx) } }() @@ -176,7 +172,18 @@ func (b *EthereumRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOu if b.Mempool == nil { return errors.New("Mempool not created") } + + // get initial mempool transactions + txs, err := b.GetMempoolTransactions() + if err != nil { + return err + } + for _, txid := range txs { + b.Mempool.AddTransactionToMempool(txid) + } + b.Mempool.OnNewTxAddr = onNewTxAddr + if b.isETC { glog.Info(b.ChainConfig.CoinName, " does not support subscription to newHeads") } else { @@ -213,6 +220,9 @@ func (b *EthereumRPC) InitializeMempool(addrDescForOutpoint bchain.AddrDescForOu }); err != nil { return err } + + b.mempoolInitialized = true + return nil } @@ -495,9 +505,9 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error return nil, errors.Annotatef(err, "hash %v, height %v, txid %v", hash, height, tx.Hash) } btxs[i] = *btx - b.pendingTransactionsLock.Lock() - delete(b.pendingTransactions, tx.Hash) - b.pendingTransactionsLock.Unlock() + if b.mempoolInitialized { + b.Mempool.RemoveTransactionFromMempool(tx.Hash) + } } bbk := bchain.Block{ BlockHeader: *bbh, @@ -535,14 +545,7 @@ func (b *EthereumRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { // GetTransactionForMempool returns a transaction by the transaction ID. // It could be optimized for mempool, i.e. without block time and confirmations func (b *EthereumRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { - tx, err := b.GetTransaction(txid) - // if there is an error getting the tx or the tx is confirmed, remove it from pending transactions - if err == bchain.ErrTxNotFound || (tx != nil && tx.Confirmations > 0) { - b.pendingTransactionsLock.Lock() - delete(b.pendingTransactions, txid) - b.pendingTransactionsLock.Unlock() - } - return tx, err + return b.GetTransaction(txid) } // GetTransaction returns a transaction by the transaction ID. @@ -555,6 +558,9 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { if err != nil { return nil, err } else if tx == nil { + if b.mempoolInitialized { + b.Mempool.RemoveTransactionFromMempool(txid) + } return nil, bchain.ErrTxNotFound } var btx *bchain.Tx @@ -621,6 +627,10 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } + // remove tx from mempool if it is there + if b.mempoolInitialized { + b.Mempool.RemoveTransactionFromMempool(txid) + } } return btx, nil } @@ -654,19 +664,7 @@ func (b *EthereumRPC) GetMempoolTransactions() ([]string, error) { return nil, err } } - b.pendingTransactionsLock.Lock() - // join transactions returned by getBlockRaw with pendingTransactions from subscription - for _, txid := range body.Transactions { - b.pendingTransactions[txid] = struct{}{} - } - txids := make([]string, len(b.pendingTransactions)) - i := 0 - for txid := range b.pendingTransactions { - txids[i] = txid - i++ - } - b.pendingTransactionsLock.Unlock() - return txids, nil + return body.Transactions, nil } // EstimateFee returns fee estimation diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index 315f166a..52f176c5 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -6,14 +6,25 @@ import ( "github.com/golang/glog" ) +const mempoolTimeoutTime = 24 * time.Hour +const mempoolTimeoutRunPeriod = 10 * time.Minute + // MempoolEthereumType is mempool handle of EthereumType chains type MempoolEthereumType struct { BaseMempool + nextTimeoutRun time.Time } // NewMempoolEthereumType creates new mempool handler. func NewMempoolEthereumType(chain BlockChain) *MempoolEthereumType { - return &MempoolEthereumType{BaseMempool: BaseMempool{chain: chain}} + return &MempoolEthereumType{ + BaseMempool: BaseMempool{ + chain: chain, + txEntries: make(map[string]txEntry), + addrDescToTx: make(map[string][]Outpoint), + }, + nextTimeoutRun: time.Now().Add(mempoolTimeoutTime), + } } func appendAddress(io []addrIndex, i int32, a string, parser BlockChainParser) []addrIndex { @@ -76,35 +87,80 @@ func (m *MempoolEthereumType) createTxEntry(txid string, txTime uint32) (txEntry return txEntry{addrIndexes: addrIndexes, time: txTime}, true } -// Resync gets mempool transactions and maps outputs to transactions. -// Resync is not reentrant, it should be called from a single thread. -// Read operations (GetTransactions) are safe. +// Resync ethereum type removes timed out transactions and returns number of transactions in mempool. +// Transactions are added/removed by AddTransactionToMempool/RemoveTransactionFromMempool methods func (m *MempoolEthereumType) Resync() (int, error) { - start := time.Now() - glog.V(1).Info("Mempool: resync") - txs, err := m.chain.GetMempoolTransactions() - if err != nil { - return 0, err - } - // allocate slightly larger capacity of the maps - newTxEntries := make(map[string]txEntry, len(m.txEntries)+5) - newAddrDescToTx := make(map[string][]Outpoint, len(m.addrDescToTx)+5) - txTime := uint32(time.Now().Unix()) - var ok bool - for _, txid := range txs { - entry, exists := m.txEntries[txid] - if !exists { - entry, ok = m.createTxEntry(txid, txTime) - if !ok { - continue + m.mux.Lock() + entries := len(m.txEntries) + now := time.Now() + if m.nextTimeoutRun.Before(now) { + threshold := now.Add(-mempoolTimeoutTime) + for txid, entry := range m.txEntries { + if time.Unix(int64(entry.time), 0).Before(threshold) { + m.removeEntryFromMempool(txid, entry) } } - newTxEntries[txid] = entry + removed := entries - len(m.txEntries) + entries = len(m.txEntries) + glog.Info("Mempool: cleanup, removed ", removed, " transactions from mempool") + m.nextTimeoutRun = now.Add(mempoolTimeoutRunPeriod) + } + m.mux.Unlock() + glog.Info("Mempool: resync ", entries, " transactions in mempool") + return entries, nil +} + +// AddTransactionToMempool adds transactions to mempool +func (m *MempoolEthereumType) AddTransactionToMempool(txid string) { + m.mux.Lock() + _, exists := m.txEntries[txid] + m.mux.Unlock() + if glog.V(1) { + glog.Info("AddTransactionToMempool ", txid, ", existed ", exists) + } + if !exists { + entry, ok := m.createTxEntry(txid, uint32(time.Now().Unix())) + if !ok { + return + } + m.mux.Lock() + m.txEntries[txid] = entry for _, si := range entry.addrIndexes { - newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], Outpoint{txid, si.n}) + m.addrDescToTx[si.addrDesc] = append(m.addrDescToTx[si.addrDesc], Outpoint{txid, si.n}) + } + m.mux.Unlock() + } +} + +func (m *MempoolEthereumType) removeEntryFromMempool(txid string, entry txEntry) { + delete(m.txEntries, txid) + for _, si := range entry.addrIndexes { + outpoints, found := m.addrDescToTx[si.addrDesc] + if found { + newOutpoints := make([]Outpoint, 0, len(outpoints)-1) + for _, o := range outpoints { + if o.Txid != txid { + newOutpoints = append(newOutpoints, o) + } + } + if len(newOutpoints) > 0 { + m.addrDescToTx[si.addrDesc] = newOutpoints + } else { + delete(m.addrDescToTx, si.addrDesc) + } } } - m.updateMappings(newTxEntries, newAddrDescToTx) - glog.Info("Mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") - return len(m.txEntries), nil +} + +// RemoveTransactionFromMempool removes transaction from mempool +func (m *MempoolEthereumType) RemoveTransactionFromMempool(txid string) { + m.mux.Lock() + entry, exists := m.txEntries[txid] + if glog.V(1) { + glog.Info("RemoveTransactionFromMempool ", txid, ", existed ", exists) + } + if exists { + m.removeEntryFromMempool(txid, entry) + } + m.mux.Unlock() } From b227a8e77703eeda4df45f7946d938fe1b35a902 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 5 Apr 2019 16:04:26 +0200 Subject: [PATCH 24/39] Modify bitcoin type mempool resync to preserve first seen time order --- bchain/basemempool.go | 23 +++++++++++++++----- bchain/mempool_bitcoin_type.go | 38 ++++++++++++++++++++------------- bchain/mempool_ethereum_type.go | 20 ----------------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/bchain/basemempool.go b/bchain/basemempool.go index 807f1ed6..68c09331 100644 --- a/bchain/basemempool.go +++ b/bchain/basemempool.go @@ -65,11 +65,24 @@ func (a MempoolTxidEntries) Less(i, j int) bool { return hi > hj } -func (m *BaseMempool) updateMappings(newTxEntries map[string]txEntry, newAddrDescToTx map[string][]Outpoint) { - m.mux.Lock() - m.txEntries = newTxEntries - m.addrDescToTx = newAddrDescToTx - m.mux.Unlock() +func (m *BaseMempool) removeEntryFromMempool(txid string, entry txEntry) { + delete(m.txEntries, txid) + for _, si := range entry.addrIndexes { + outpoints, found := m.addrDescToTx[si.addrDesc] + if found { + newOutpoints := make([]Outpoint, 0, len(outpoints)-1) + for _, o := range outpoints { + if o.Txid != txid { + newOutpoints = append(newOutpoints, o) + } + } + if len(newOutpoints) > 0 { + m.addrDescToTx[si.addrDesc] = newOutpoints + } else { + delete(m.addrDescToTx, si.addrDesc) + } + } + } } // GetAllEntries returns all mempool entries sorted by fist seen time in descending order diff --git a/bchain/mempool_bitcoin_type.go b/bchain/mempool_bitcoin_type.go index 929dd26c..68882886 100644 --- a/bchain/mempool_bitcoin_type.go +++ b/bchain/mempool_bitcoin_type.go @@ -19,7 +19,9 @@ type MempoolBitcoinType struct { func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *MempoolBitcoinType { m := &MempoolBitcoinType{ BaseMempool: BaseMempool{ - chain: chain, + chain: chain, + txEntries: make(map[string]txEntry), + addrDescToTx: make(map[string][]Outpoint), }, chanTxid: make(chan string, 1), chanAddrIndex: make(chan txidio, 1), @@ -137,29 +139,30 @@ func (m *MempoolBitcoinType) Resync() (int, error) { return 0, err } glog.V(2).Info("mempool: resync ", len(txs), " txs") - // allocate slightly larger capacity of the maps - newTxEntries := make(map[string]txEntry, len(m.txEntries)+5) - newAddrDescToTx := make(map[string][]Outpoint, len(m.addrDescToTx)+5) - dispatched := 0 - txTime := uint32(time.Now().Unix()) - onNewData := func(txid string, entry txEntry) { + onNewEntry := func(txid string, entry txEntry) { if len(entry.addrIndexes) > 0 { - newTxEntries[txid] = entry + m.mux.Lock() + m.txEntries[txid] = entry for _, si := range entry.addrIndexes { - newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], Outpoint{txid, si.n}) + m.addrDescToTx[si.addrDesc] = append(m.addrDescToTx[si.addrDesc], Outpoint{txid, si.n}) } + m.mux.Unlock() } } + txsMap := make(map[string]struct{}, len(txs)) + dispatched := 0 + txTime := uint32(time.Now().Unix()) // get transaction in parallel using goroutines created in NewUTXOMempool for _, txid := range txs { - io, exists := m.txEntries[txid] + txsMap[txid] = struct{}{} + _, exists := m.txEntries[txid] if !exists { loop: for { select { // store as many processed transactions as possible case tio := <-m.chanAddrIndex: - onNewData(tio.txid, txEntry{tio.io, txTime}) + onNewEntry(tio.txid, txEntry{tio.io, txTime}) dispatched-- // send transaction to be processed case m.chanTxid <- txid: @@ -167,15 +170,20 @@ func (m *MempoolBitcoinType) Resync() (int, error) { break loop } } - } else { - onNewData(txid, io) } } for i := 0; i < dispatched; i++ { tio := <-m.chanAddrIndex - onNewData(tio.txid, txEntry{tio.io, txTime}) + onNewEntry(tio.txid, txEntry{tio.io, txTime}) + } + + for txid, entry := range m.txEntries { + if _, exists := txsMap[txid]; !exists { + m.mux.Lock() + m.removeEntryFromMempool(txid, entry) + m.mux.Unlock() + } } - m.updateMappings(newTxEntries, newAddrDescToTx) glog.Info("mempool: resync finished in ", time.Since(start), ", ", len(m.txEntries), " transactions in mempool") return len(m.txEntries), nil } diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index 52f176c5..4eb5b969 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -132,26 +132,6 @@ func (m *MempoolEthereumType) AddTransactionToMempool(txid string) { } } -func (m *MempoolEthereumType) removeEntryFromMempool(txid string, entry txEntry) { - delete(m.txEntries, txid) - for _, si := range entry.addrIndexes { - outpoints, found := m.addrDescToTx[si.addrDesc] - if found { - newOutpoints := make([]Outpoint, 0, len(outpoints)-1) - for _, o := range outpoints { - if o.Txid != txid { - newOutpoints = append(newOutpoints, o) - } - } - if len(newOutpoints) > 0 { - m.addrDescToTx[si.addrDesc] = newOutpoints - } else { - delete(m.addrDescToTx, si.addrDesc) - } - } - } -} - // RemoveTransactionFromMempool removes transaction from mempool func (m *MempoolEthereumType) RemoveTransactionFromMempool(txid string) { m.mux.Lock() From 4b0addbe989b5dacf91957203b1aab12cca395f1 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 5 Apr 2019 17:13:56 +0200 Subject: [PATCH 25/39] Preserve socket.io BlockTimestamp backward compatibility --- server/socketio.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/socketio.go b/server/socketio.go index a1790762..4b350f27 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -328,13 +328,15 @@ func txToResTx(tx *api.Tx) resTx { outputs[i] = output } var h int + var blocktime int64 if tx.Confirmations == 0 { h = -1 } else { h = int(tx.Blockheight) + blocktime = tx.Blocktime } return resTx{ - BlockTimestamp: tx.Blocktime, + BlockTimestamp: blocktime, FeeSatoshis: tx.FeesSat.AsInt64(), Hash: tx.Txid, Height: h, From e394350e85d632901ad8a5c43e33b156c5db260a Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 5 Apr 2019 17:14:08 +0200 Subject: [PATCH 26/39] Update api documentation --- docs/api.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/api.md b/docs/api.md index b028268d..728ec337 100644 --- a/docs/api.md +++ b/docs/api.md @@ -167,6 +167,10 @@ Response for Ethereum-type coins. There is always only one *vin*, only one *vout } ``` +A note about the `blocktime` field: +- for already mined transaction (`confirmations > 0`), the field `blocktime` contains time of the block +- for transactions in mempool (`confirmations == 0`), the field contains time when the running instance of Blockbook was first time notified about the transaction. This time may be different in different instances of Blockbook. + #### Get transaction specific Returns transaction data in the exact format as returned by backend, including all coin specific fields: From 127d24be06fea659b2ce3df5bdfd475d527223d2 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 5 Apr 2019 17:32:26 +0200 Subject: [PATCH 27/39] Give option to show derived xpub addresses in explorer --- static/templates/xpub.html | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/static/templates/xpub.html b/static/templates/xpub.html index df324738..dfe196b8 100644 --- a/static/templates/xpub.html +++ b/static/templates/xpub.html @@ -29,8 +29,8 @@ Used XPUB Addresses {{$addr.TotalTokens}} - {{- if or $addr.Tokens $addr.TotalTokens -}} - + + {{- if or $addr.Tokens $addr.TotalTokens -}} {{if $data.NonZeroBalanceTokens}}XPUB Addresses with Balance{{else}}XPUB Addresses{{end}} @@ -51,15 +51,17 @@ {{- end -}} {{- if $data.NonZeroBalanceTokens -}} - + {{- end -}}
Show all XPUB addressesShow used XPUB addressesShow derived XPUB addresses
+ {{- else -}} + Show derived XPUB addresses + {{- end -}} - {{- end -}} - +
From f108559d7c17e4c6beac9c5301a67317bd011aac Mon Sep 17 00:00:00 2001 From: Cronos <37080958+eabz@users.noreply.github.com> Date: Fri, 5 Apr 2019 16:18:24 -0600 Subject: [PATCH 28/39] Support Polis (#118) * Update .gitignore Intelij IDEA * Polis initial configuration * Remove polis-qt from excluded files * Fix * Update v1.4.10 * Test files * Fix * Add PackedTxInfo * Add Parsing Blocks test * Exclude polis-qt * Integration test data * Fix --- .gitignore | 3 +- bchain/coins/blockchain.go | 2 + bchain/coins/polis/polisparser.go | 78 +++++ bchain/coins/polis/polisparser_test.go | 308 ++++++++++++++++++ bchain/coins/polis/polisrpc.go | 56 ++++ bchain/coins/polis/testdata/block_dump.280000 | 1 + bchain/coins/polis/testdata/block_dump.50000 | 1 + configs/coins/polis.json | 66 ++++ docs/ports.md | 1 + tests/rpc/testdata/polis.json | 40 +++ tests/sync/testdata/polis.json | 78 +++++ tests/tests.json | 5 + 12 files changed, 638 insertions(+), 1 deletion(-) create mode 100644 bchain/coins/polis/polisparser.go create mode 100644 bchain/coins/polis/polisparser_test.go create mode 100644 bchain/coins/polis/polisrpc.go create mode 100644 bchain/coins/polis/testdata/block_dump.280000 create mode 100644 bchain/coins/polis/testdata/block_dump.50000 create mode 100644 configs/coins/polis.json create mode 100644 tests/rpc/testdata/polis.json create mode 100644 tests/sync/testdata/polis.json diff --git a/.gitignore b/.gitignore index 8d866cba..23e1078c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ build/ldb build/sst_dump build/*.deb .bin-image -.deb-image \ No newline at end of file +.deb-image +\.idea/ diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 7b39eae4..c4662e4a 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -22,6 +22,7 @@ import ( "blockbook/bchain/coins/namecoin" "blockbook/bchain/coins/nuls" "blockbook/bchain/coins/pivx" + "blockbook/bchain/coins/polis" "blockbook/bchain/coins/qtum" "blockbook/bchain/coins/vertcoin" "blockbook/bchain/coins/xzc" @@ -74,6 +75,7 @@ func init() { BlockChainFactories["Groestlcoin Testnet"] = grs.NewGroestlcoinRPC BlockChainFactories["PIVX"] = pivx.NewPivXRPC BlockChainFactories["PIVX Testnet"] = pivx.NewPivXRPC + BlockChainFactories["Polis"] = polis.NewPolisRPC BlockChainFactories["Zcoin"] = xzc.NewZcoinRPC BlockChainFactories["Fujicoin"] = fujicoin.NewFujicoinRPC BlockChainFactories["Flo"] = flo.NewFloRPC diff --git a/bchain/coins/polis/polisparser.go b/bchain/coins/polis/polisparser.go new file mode 100644 index 00000000..7b56904b --- /dev/null +++ b/bchain/coins/polis/polisparser.go @@ -0,0 +1,78 @@ +package polis + +import ( + "blockbook/bchain/coins/btc" + "github.com/martinboehm/btcd/wire" + "github.com/martinboehm/btcutil/chaincfg" +) + +const ( + MainnetMagic wire.BitcoinNet = 0xbd6b0cbf + TestnetMagic wire.BitcoinNet = 0xffcae2ce + RegtestMagic wire.BitcoinNet = 0xdcb7c1fc +) + +var ( + MainNetParams chaincfg.Params + TestNetParams chaincfg.Params + RegtestParams chaincfg.Params +) + +func init() { + MainNetParams = chaincfg.MainNetParams + MainNetParams.Net = MainnetMagic + + // Address encoding magics + MainNetParams.PubKeyHashAddrID = []byte{55} // base58 prefix: P + MainNetParams.ScriptHashAddrID = []byte{56} // base58 prefix: 3 + + TestNetParams = chaincfg.TestNet3Params + TestNetParams.Net = TestnetMagic + + // Address encoding magics + TestNetParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + TestNetParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 + + RegtestParams = chaincfg.RegressionNetParams + RegtestParams.Net = RegtestMagic + + // Address encoding magics + RegtestParams.PubKeyHashAddrID = []byte{140} // base58 prefix: y + RegtestParams.ScriptHashAddrID = []byte{19} // base58 prefix: 8 or 9 +} + +// PolisParser handle +type PolisParser struct { + *btc.BitcoinParser +} + +// NewPolisParser returns new PolisParser instance +func NewPolisParser(params *chaincfg.Params, c *btc.Configuration) *PolisParser { + return &PolisParser{BitcoinParser: btc.NewBitcoinParser(params, c)} +} + +// GetChainParams contains network parameters for the main Polis network, +// the regression test Polis network, the test Polis network and +// the simulation test Polis network, in this order +func GetChainParams(chain string) *chaincfg.Params { + if !chaincfg.IsRegistered(&MainNetParams) { + err := chaincfg.Register(&MainNetParams) + if err == nil { + err = chaincfg.Register(&TestNetParams) + } + if err == nil { + err = chaincfg.Register(&RegtestParams) + } + if err != nil { + panic(err) + } + } + switch chain { + case "test": + return &TestNetParams + case "regtest": + return &RegtestParams + default: + return &MainNetParams + } +} diff --git a/bchain/coins/polis/polisparser_test.go b/bchain/coins/polis/polisparser_test.go new file mode 100644 index 00000000..b0888dda --- /dev/null +++ b/bchain/coins/polis/polisparser_test.go @@ -0,0 +1,308 @@ +// +build unittest + +package polis + +import ( + "bytes" + "fmt" + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "encoding/hex" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "reflect" + "testing" + "github.com/martinboehm/btcutil/chaincfg" +) + + +type testBlock struct { + size int + time int64 + txs []string +} + +var testParseBlockTxs = map[int]testBlock{ + // Simple POW block + 50000: { + size: 1393, + time: 1520175937, + txs: []string{ + "b68057244d6ad2df0017bad1cd8a24487e21404b52873c59876a484c93f5a69e", + "3d27b82972196ecce604c2923bc9105cd683352fd56f8ba052dee57a296a2d71", + "d22704ddad675d652a6b501694d9e026f19b41842946ea285490315cb944c674", + "7be2e53414c9480ea1590e40cfa8361ac7594f7435f919a6eeff96367b0dffa9", + "e2486a9610698888c4baad7001385e95aca053ab9fc7cc9d15280c9c835c975c", + }, + }, + // Simple POS block + 280000: { + size: 275, + time: 1549070495, + txs: []string{ + "9a820cb226364e852ec5d13bc3ead1ad127bf28ef2808919571200a1262b46b5", + "fcca99e281fa0c43085dfe82c24b4367ff21d3a148539e781c061fe29a793ab1", + }, + }, +} + +func TestMain(m *testing.M) { + c := m.Run() + chaincfg.ResetParams() + os.Exit(c) +} + +func helperLoadBlock(t *testing.T, height int) []byte { + name := fmt.Sprintf("block_dump.%d", height) + path := filepath.Join("testdata", name) + + d, err := ioutil.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + d = bytes.TrimSpace(d) + + b := make([]byte, hex.DecodedLen(len(d))) + _, err = hex.Decode(b, d) + if err != nil { + t.Fatal(err) + } + + return b +} + +func TestParseBlock(t *testing.T) { + p := NewPolisParser(GetChainParams("main"), &btc.Configuration{}) + + for height, tb := range testParseBlockTxs { + b := helperLoadBlock(t, height) + + blk, err := p.ParseBlock(b) + if err != nil { + t.Errorf("ParseBlock() error %v", err) + } + + if blk.Size != tb.size { + t.Errorf("ParseBlock() block size: got %d, want %d", blk.Size, tb.size) + } + + if blk.Time != tb.time { + t.Errorf("ParseBlock() block time: got %d, want %d", blk.Time, tb.time) + } + + if len(blk.Txs) != len(tb.txs) { + t.Errorf("ParseBlock() number of transactions: got %d, want %d", len(blk.Txs), len(tb.txs)) + } + + for ti, tx := range tb.txs { + if blk.Txs[ti].Txid != tx { + t.Errorf("ParseBlock() transaction %d: got %s, want %s", ti, blk.Txs[ti].Txid, tx) + } + } + } +} + +func Test_GetAddrDescFromAddress_Mainnet(t *testing.T) { + type args struct { + address string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "P2PKH1", + args: args{address: "P9hRjWq6tMqhroxswc2f5jp2ND2py8YEnu"}, + want: "76a9140c26ca7967e6fe946f00bf81bcd3b86f43538edf88ac", + wantErr: false, + }, + } + parser := NewPolisParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parser.GetAddrDescFromAddress(tt.args.address) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) + } + }) + } +} + +func Test_GetAddressesFromAddrDesc(t *testing.T) { + type args struct { + script string + } + tests := []struct { + name string + args args + want []string + want2 bool + wantErr bool + }{ + { + name: "P2PKH1", + args: args{script: "76a9140c26ca7967e6fe946f00bf81bcd3b86f43538edf88ac"}, + want: []string{"P9hRjWq6tMqhroxswc2f5jp2ND2py8YEnu"}, + want2: true, + wantErr: false, + }, + } + + parser := NewPolisParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.script) + got, got2, err := parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) + } + }) + } +} + +var ( + testTx1 bchain.Tx + testTxPacked1 = "0004e3868bca91b06e020000000198160d0ba0168003897358f1a6d2a2499a8e93dc6d341613b960ed2083de3fe0010000006b483045022100c531bd672ed3cb1a9285191ac168f1627e6075a54f5cf4a22ee8ec717b9a047f0220068241d9cc10adf1ddcc30aef4e62863d663af50371ebc639d11532ddf6e636e012102182bc5cd0a82c43c7ed9c4acc4b735e04e7f3275b43ff2514b8a1beb1feb5493feffffff021e470d8d0e0000001976a914344bf2db193190967d3b8da659a3ce2fde5f44a588acb63dd7a4300000001976a91414c343cae45bbcf7a27b8284b8c328587f6cc45588ac85e30400" +) + +func init() { + testTx1 = bchain.Tx{ + Hex: "020000000198160d0ba0168003897358f1a6d2a2499a8e93dc6d341613b960ed2083de3fe0010000006b483045022100c531bd672ed3cb1a9285191ac168f1627e6075a54f5cf4a22ee8ec717b9a047f0220068241d9cc10adf1ddcc30aef4e62863d663af50371ebc639d11532ddf6e636e012102182bc5cd0a82c43c7ed9c4acc4b735e04e7f3275b43ff2514b8a1beb1feb5493feffffff021e470d8d0e0000001976a914344bf2db193190967d3b8da659a3ce2fde5f44a588acb63dd7a4300000001976a91414c343cae45bbcf7a27b8284b8c328587f6cc45588ac85e30400", + Blocktime: 1554132023, + Txid: "6882e77c916c5442d09e295b88fbb8a2fac6dbb988975bb00dbded088e0229a9", + LockTime: 320389, + Version: 2, + Vin: []bchain.Vin{ + { + ScriptSig: bchain.ScriptSig{ + Hex: "483045022100c531bd672ed3cb1a9285191ac168f1627e6075a54f5cf4a22ee8ec717b9a047f0220068241d9cc10adf1ddcc30aef4e62863d663af50371ebc639d11532ddf6e636e012102182bc5cd0a82c43c7ed9c4acc4b735e04e7f3275b43ff2514b8a1beb1feb5493", + }, + Txid: "e03fde8320ed60b91316346ddc938e9a49a2d2a6f1587389038016a00b0d1698", + Vout: 1, + Sequence: 4294967294, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(62495999774), + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "76a914344bf2db193190967d3b8da659a3ce2fde5f44a588ac", + Addresses: []string{ + "PDMhGxFYTaomhzSqWKHbUzx7smYUZvZVjd", + }, + }, + }, + { + ValueSat: *big.NewInt(208923999670), + N: 1, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "76a91414c343cae45bbcf7a27b8284b8c328587f6cc45588ac", + Addresses: []string{ + "PAUxb3g3DZNrjgbRidZy3NC9TNhVrPRzAR", + }, + }, + }, + }, + } +} + +func Test_PackTx(t *testing.T) { + type args struct { + tx bchain.Tx + height uint32 + blockTime int64 + parser *PolisParser + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "polis-1", + args: args{ + tx: testTx1, + height: 320390, + blockTime: 1554132023, + parser: NewPolisParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: testTxPacked1, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.parser.PackTx(&tt.args.tx, tt.args.height, tt.args.blockTime) + if (err != nil) != tt.wantErr { + t.Errorf("packTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("packTx() = %v, want %v", h, tt.want) + } + }) + } +} + +func Test_UnpackTx(t *testing.T) { + type args struct { + packedTx string + parser *PolisParser + } + tests := []struct { + name string + args args + want *bchain.Tx + want1 uint32 + wantErr bool + }{ + { + name: "polis-1", + args: args{ + packedTx: testTxPacked1, + parser: NewPolisParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: &testTx1, + want1: 320390, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.packedTx) + got, got1, err := tt.args.parser.UnpackTx(b) + if (err != nil) != tt.wantErr { + t.Errorf("unpackTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unpackTx() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("unpackTx() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/bchain/coins/polis/polisrpc.go b/bchain/coins/polis/polisrpc.go new file mode 100644 index 00000000..aa31f22e --- /dev/null +++ b/bchain/coins/polis/polisrpc.go @@ -0,0 +1,56 @@ +package polis + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "encoding/json" + + "github.com/golang/glog" +) + +// PolisRPC is an interface to JSON-RPC bitcoind service. +type PolisRPC struct { + *btc.BitcoinRPC +} + +// NewPolisRPC returns new PolisRPC instance. +func NewPolisRPC(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) { + b, err := btc.NewBitcoinRPC(config, pushHandler) + if err != nil { + return nil, err + } + + s := &PolisRPC{ + b.(*btc.BitcoinRPC), + } + s.RPCMarshaler = btc.JSONMarshalerV1{} + s.ChainConfig.SupportsEstimateSmartFee = false + + return s, nil +} + +// Initialize initializes PolisRPC instance. +func (b *PolisRPC) Initialize() error { + chainName, err := b.GetChainInfoAndInitializeMempool(b) + if err != nil { + return err + } + + params := GetChainParams(chainName) + + // always create parser + b.Parser = NewPolisParser(params, b.ChainConfig) + + // parameters for getInfo request + if params.Net == MainnetMagic { + b.Testnet = false + b.Network = "livenet" + } else { + b.Testnet = true + b.Network = "testnet" + } + + glog.Info("rpc: block chain ", params.Name) + + return nil +} diff --git a/bchain/coins/polis/testdata/block_dump.280000 b/bchain/coins/polis/testdata/block_dump.280000 new file mode 100644 index 00000000..7382ea24 --- /dev/null +++ b/bchain/coins/polis/testdata/block_dump.280000 @@ -0,0 +1 @@ +00000020dfc2afd28e72a76db82a40793e13f5fb1f7a2b95c81bf1c388129cdbc3052f0f53556fd7ecd06002359e20f52a6b9bd08f94d2e4441256499ce46e91741dbb7e9ff0545c56b24c1c000000000202000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0603c045040101ffffffff01000000000000000000000000000200000001bc90c3dcec40ef8855c547d60145db719265e17067e9a9f9d1c94a38b0a8a4c30200000000ffffffff030000000000000000000030ff66000000001976a914ba364eade2f5a3b6068c30fcde08378eaa2ff29e88ac0020aa44000000001976a9147d7c5860b78c814f802012563eb79dc398b3b14688ac00000000 \ No newline at end of file diff --git a/bchain/coins/polis/testdata/block_dump.50000 b/bchain/coins/polis/testdata/block_dump.50000 new file mode 100644 index 00000000..c2eb0361 --- /dev/null +++ b/bchain/coins/polis/testdata/block_dump.50000 @@ -0,0 +1 @@ +00000020df750da1e6ef1d07bacc08c386e7059b67bfb381395c7dfc9703000000000000f3f3e5d5ef207575e353dd7a1a59f18473a35a77419c5afae361d55709ad7892410b9c5ac176061a6c6b09400501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff200350c30004410b9c5a0840056105567b02000d2f6e6f64655374726174756d2f0000000002f02a7515000000001976a9146629c9b161656be9637e0dd83153d7600084047b88acbfabd455000000001976a914f59d4ec7db19d17d6c43e75376ebb3848e0a24d288ac000000000100000003db65905855874c10d72d5a9b9e3fa7d541dae2240c9fe98028c63391847e930b010000006a473044022058fb25986162a2b59412d810b8ffe6100892599f6ecf82ce4293743ccc175c8f0220191855fc2ac4047db37784d5465a701a2d28f1f549bf359a26f684c9660d6cd40121036bb85d56ee2010121836af34dc42c4452895cddbfb5efb1211cde91cf267ef52feffffffe92f932936c37cc66e5cf2abe5e970ee2d64ec7873f1b06b7587f609ad6f5b6a010000006a47304402204d0159f20fee14b457f05e03cdacf0dcdbac332dbacd81712bb14e0c372450c4022022806cb0b8940f43a4af18a57b12c576fe135c1388eddc9e64e72875dba314e60121036bb85d56ee2010121836af34dc42c4452895cddbfb5efb1211cde91cf267ef52feffffffd38d7da8cb5576d58388e0d11a4c961a8a62ef7868488fe0a716139c0feffebd010000006b483045022100f2e12113349751239636e02c246bd4a6c636b2cc41e4e1dc3ec5a6db7fa02d0a022030c93a94c317a8ec1cf7ee9d437033b3c6dd97fa4b0deaeaa82cd9421941dfbb0121036bb85d56ee2010121836af34dc42c4452895cddbfb5efb1211cde91cf267ef52feffffff01c366b963000000001976a9141e1b579b13f646679ef8de4fee795830ea58d0f488ac4fc300000100000001f29f1bbf47d7052547f95413e81d6f9114a0d12e7cfc6272badcfaae672c27a9010000006a47304402202815ea3f59923df9d47ce70bc94387e838b43c2aa12a449222ede5849955db09022069588b99110035946dfb85f53be4316f21ed5e3321699e0d1c64ce2c07c60a2c012103424311c02421c52480177a3ecdc4c97d580798862fd490e821540b571abf7f79feffffff0260a50906000000001976a914a605e54f7c07dd797009e32d950f4d019ff5096e88ac3034040d000000001976a914af26b84931364f7eafbf86c5e4604e8b7e64a59d88ac4fc300000100000001f596f3280ee23d9f812a0983511e9ebe3508cfd6840a5810fe290a68dddff5d6000000006a47304402200ffe565761b4494b4d1cd8c535e816795f4f2be24f9667f454ab1eb905375fdb02202eb41f281cdd0f1ed17824c72926fe56c7508f93c2048ed5832913befa24ad2501210331ef0b98c08f2d0dbadbfb1262a330046e667f74d3a2d0428d51cba33cd068e8feffffff0278a62900000000001976a91495ec2efe8ce15ad80f92290cfa20f54e024f1d6388ac87acd902000000001976a914b024d8793d072b989b2d2f636f2494e98190bb5d88ac4fc300000100000001b0a5f838b6fd3d51ce2b48fc2c35fb4519be6332faf8979ba97d559727788bce000000006a4730440220089e42b35c10187e2e250abb4a448e044a0f24d7c7043b7f70acc54ec306903802206524b755d784ccadd9991a42e782412320566da925fb2492e6209be07d9a0011012102e9d115035383d4c97982642134104cdf580ea38eedbfeb2f836ef727e8cec869feffffff027c9c4d00000000001976a914ca23ff9efd82b1b0f2767573621908077bd9840588ac9537f007000000001976a914aedba9af0407cc64fbd4d3e9e715cc46e72105c088ac4fc30000 \ No newline at end of file diff --git a/configs/coins/polis.json b/configs/coins/polis.json new file mode 100644 index 00000000..18fd67b1 --- /dev/null +++ b/configs/coins/polis.json @@ -0,0 +1,66 @@ +{ + "coin": { + "name": "Polis", + "shortcut": "POLIS", + "label": "Polis", + "alias": "polis" + }, + "ports": { + "backend_rpc": 8067, + "backend_message_queue": 38367, + "blockbook_internal": 9067, + "blockbook_public": 9167 + }, + "ipc": { + "rpc_url_template": "http://127.0.0.1:{{.Ports.BackendRPC}}", + "rpc_user": "rpc", + "rpc_pass": "rpc", + "rpc_timeout": 25, + "message_queue_binding_template": "tcp://127.0.0.1:{{.Ports.BackendMessageQueue}}" + }, + "backend": { + "package_name": "backend-polis", + "package_revision": "satoshilabs-1", + "system_user": "polis", + "version": "1.4.10", + "binary_url": "https://github.com/polispay/polis/releases/download/v1.4.10/poliscore-1.4.10-x86_64-linux-gnu.tar.gz", + "verification_type": "sha256", + "verification_source": "9688beeb4c789c2b20f4277f7363fcc986fb0e6cc75127179663272642fafcd3", + "extract_command": "tar -C backend --strip 1 -xf", + "exclude_files": [ + "bin/polis-qt" + ], + "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/polisd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/*.log", + "postinst_script_template": "", + "service_type": "forking", + "service_additional_params_template": "", + "protect_memory": true, + "mainnet": true, + "server_config_file": "bitcoin_like.conf", + "client_config_file": "bitcoin_like_client.conf", + "additional_params": { + "mempoolexpiry": 72 + } + }, + "blockbook": { + "package_name": "blockbook-polis", + "system_user": "blockbook-polis", + "internal_binding_template": ":{{.Ports.BlockbookInternal}}", + "public_binding_template": ":{{.Ports.BlockbookPublic}}", + "explorer_url": "", + "additional_params": "", + "block_chain": { + "parse": true, + "subversion": "/Polis Core:1.4.10/", + "mempool_workers": 8, + "mempool_sub_workers": 2, + "block_addresses_to_keep": 300, + "additional_params": {} + } + }, + "meta": { + "package_maintainer": "Cronos", + "package_maintainer_email": "eabz@polispay.org" + } +} diff --git a/docs/ports.md b/docs/ports.md index 91f5c727..84e5eaf5 100644 --- a/docs/ports.md +++ b/docs/ports.md @@ -27,6 +27,7 @@ | Bellcoin | 9052 | 9152 | 8052 | 38352 | | NULS | 9053 | 9153 | 8053 | 38353 | | Flo | 9066 | 9166 | 8066 | 38366 | +| Polis | 9067 | 9167 | 8067 | 38367 | | Qtum | 9088 | 9188 | 8088 | 38388 | | Bitcoin Testnet | 19030 | 19130 | 18030 | 48330 | | Bitcoin Cash Testnet | 19031 | 19131 | 18031 | 48331 | diff --git a/tests/rpc/testdata/polis.json b/tests/rpc/testdata/polis.json new file mode 100644 index 00000000..2de8281c --- /dev/null +++ b/tests/rpc/testdata/polis.json @@ -0,0 +1,40 @@ +{ + "blockHeight": 308506, + "blockHash": "7db10f77c97acf921cb57843b0155657b57de674cd914802ce3320f52d478dfe", + "blockTime": 1552686920, + "blockTxs": [ + "319a7c8bcec7951931b79eade2985362c41b827f3fd013dedfe9b1a96fa57a03", + "7780a2a8fb2356ecfaaf556ff53ba93a46cdce3bd93be8256a7e304f404600bc", + "17f243fa49f5316062d650b20ce37d6dd795e46ed9fd4f64c9b4ccf689915d9b", + "72270d839f189d1193953e1450a31b620ba255a6d664c8a9a0386f8b8e2800f1" + ], + "txDetails": { + "17f243fa49f5316062d650b20ce37d6dd795e46ed9fd4f64c9b4ccf689915d9b": { + "hex": "0200000001adc444b348621c06d881f6d1402fcb188316163c0c510d845b71cfc3a9ee1101010000006b483045022100e8378228debfddf315ebbc4a7455bfa3fceb8ae577c176e760f73c4be9ba781702201081481c27b72a9d2f8826f20e6289323276288a96181866821111c805d9305701210230ffa39fe697cc4d28d453ed84ed6545b9ce83a0863367b831dae54674488967ffffffff01f89bb0b0000000001976a914a738d8ab602e40c83c1d617aa3075b1954312d7988ac00000000", + "txid": "17f243fa49f5316062d650b20ce37d6dd795e46ed9fd4f64c9b4ccf689915d9b", + "blocktime": 1552686920, + "time": 1552686920, + "locktime": 0, + "version": 2, + "vin": [ + { + "txid": "0111eea9c3cf715b840d510c3c16168318cb2f40d1f681d8061c6248b344c4ad", + "vout": 1, + "sequence": 4294967295, + "scriptSig": { + "hex": "483045022100e8378228debfddf315ebbc4a7455bfa3fceb8ae577c176e760f73c4be9ba781702201081481c27b72a9d2f8826f20e6289323276288a96181866821111c805d9305701210230ffa39fe697cc4d28d453ed84ed6545b9ce83a0863367b831dae54674488967" + } + } + ], + "vout": [ + { + "value": 29.64364280, + "n": 0, + "scriptPubKey": { + "hex": "76a914a738d8ab602e40c83c1d617aa3075b1954312d7988ac" + } + } + ] + } + } +} diff --git a/tests/sync/testdata/polis.json b/tests/sync/testdata/polis.json new file mode 100644 index 00000000..ff96ebb8 --- /dev/null +++ b/tests/sync/testdata/polis.json @@ -0,0 +1,78 @@ +{ + "connectBlocks": { + "syncRanges": [ + {"lower": 248706, "upper": 248726} + ], + "blocks": { + "300900": { + "height": 300900, + "hash": "427125bf9fe38a04cbe6e23b1ccfdd9aa9aec2e32350300e6f5e96ecd24926bb", + "noTxs": 2, + "txDetails": [ + { + "hex": "02000000010000000000000000000000000000000000000000000000000000000000000000ffffffff06036497040101ffffffff0100000000000000000000000000", + "txid": "25144a48788e97f64aebb7c9f79452be07d45d212068b83f90787f657c8f4051", + "version": 2, + "vin": [ + { + "coinbase": "036497040101", + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 0.00000000, + "n": 0, + "scriptPubKey": { + "hex": "" + } + } + ], + "time": 1551720401, + "blocktime": 1551720401 + }, + { + "hex": "02000000013cfcafa26d96e9735ae5dacbe7ab9e3e1e13c472ac63bc27ac5ee4c799d20d9f0100000000ffffffff030000000000000000006ca53bdf0a0000001976a914e744aa894a8de0f200f8a00f58d2367c7523cae788ac0020aa44000000001976a914fec8380fd18ca3ad8c1c42938e102ee3ffac6fc188ac00000000", + "txid": "5174a1980cee089ca868758ffd396d71e742f34d9c7bada21d294b54e6819a4f", + "version": 2, + "vin": [ + { + "txid": "9f0dd299c7e45eac27bc63ac72c4131e3e9eabe7cbdae55a73e9966da2affc3c", + "vout": 1, + "scriptSig": { + "hex": "" + }, + "sequence": 4294967295 + } + ], + "vout": [ + { + "value": 0.00000000, + "n": 0, + "scriptPubKey": { + "hex": "" + } + }, + { + "value": 466.94901100, + "n": 1, + "scriptPubKey": { + "hex": "76a914e744aa894a8de0f200f8a00f58d2367c7523cae788ac" + } + }, + { + "value": 11.52000000, + "n": 2, + "scriptPubKey": { + "hex": "76a914fec8380fd18ca3ad8c1c42938e102ee3ffac6fc188ac" + } + } + ], + "time": 1551720401, + "blocktime": 1551720401 + } + ] + } + } + } +} diff --git a/tests/tests.json b/tests/tests.json index 82a45ccb..69833495 100644 --- a/tests/tests.json +++ b/tests/tests.json @@ -116,6 +116,11 @@ "EstimateSmartFee", "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"], "sync": ["ConnectBlocksParallel", "ConnectBlocks", "HandleFork"] }, + "polis": { + "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", + "EstimateSmartFee", "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"], + "sync": ["ConnectBlocksParallel", "ConnectBlocks"] + }, "zcoin": { "rpc": ["GetBlock", "GetBlockHash", "GetTransaction", "GetTransactionForMempool", "MempoolSync", "EstimateFee", "GetBestBlockHash", "GetBestBlockHeight", "GetBlockHeader"], From 02c65bc67d6e7a570f4a22f1c36850e0c45313c3 Mon Sep 17 00:00:00 2001 From: Jin Eguchi Date: Mon, 8 Apr 2019 11:57:03 +0900 Subject: [PATCH 29/39] Fix link path --- docs/config.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/config.md b/docs/config.md index db4f5354..b4e14296 100644 --- a/docs/config.md +++ b/docs/config.md @@ -12,8 +12,8 @@ automatically, there is sometimes necessary see intermediate step. You can gener generated to *build/pkg-defs* directory. Good examples of coin configuration are -[*configs/coins/bitcoin.json*](configs/coins/bitcoin.json) and -[*configs/coins/ethereum.json*](configs/coins/ethereum.json) for Bitcoin-like coins and different coins, respectively. +[*configs/coins/bitcoin.json*](/configs/coins/bitcoin.json) and +[*configs/coins/ethereum.json*](/configs/coins/ethereum.json) for Bitcoin-like coins and different coins, respectively. ## Description of coin definition @@ -60,12 +60,12 @@ Good examples of coin configuration are service unit. See note on templates below. * `logrotate_files_template` – Template that define log files rotated by logrotate daemon. See note on templates below. - * `postinst_script_template` – Additional steps in postinst script. See [ZCash definition](configs/coins/zcash.json) + * `postinst_script_template` – Additional steps in postinst script. See [ZCash definition](/configs/coins/zcash.json) for more information. * `service_type` – Type of service. Services that daemonize must have *forking* type and write their PID to *PIDFile*. Services that don't support daemonization must have *simple* type. See examples above. * `service_additional_params_template` – Additional parameters in service unit. See - [ZCash definition](configs/coins/zcash.json) for more information. + [ZCash definition](/configs/coins/zcash.json) for more information. * `protect_memory` – Enables *MemoryDenyWriteExecute* option in service unit if *true*. * `mainnet` – Set *false* for testnet back-end. * `config_file` – Name of template of back-end configuration file. Templates are defined in *build/backend/config*. @@ -80,7 +80,7 @@ Good examples of coin configuration are * `internal_binding_template` – Template for *-internal* parameter. See note on templates below. * `public_binding_template` – Template for *-public* parameter. See note on templates below. * `explorer_url` – URL of blockchain explorer. Leave empty for internal explorer. - * `additional_params` – Additional params of exec command (see [Dogecoin definition](configs/coins/dogecoin.json)). + * `additional_params` – Additional params of exec command (see [Dogecoin definition](/configs/coins/dogecoin.json)). * `block_chain` – Configuration of BlockChain type that ensures communication with back-end service. All options must be tweaked for each individual coin separely. * `parse` – Use binary parser for block decoding if *true* else call verbose back-end RPC method that returns From 7ac877f16060c48411e6c9c7ce6d37e4cd63a881 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 8 Apr 2019 12:54:06 +0200 Subject: [PATCH 30/39] Apply modified BlockChain to PolisRPC --- bchain/coins/polis/polisrpc.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bchain/coins/polis/polisrpc.go b/bchain/coins/polis/polisrpc.go index aa31f22e..47e9fb7a 100644 --- a/bchain/coins/polis/polisrpc.go +++ b/bchain/coins/polis/polisrpc.go @@ -31,10 +31,11 @@ func NewPolisRPC(config json.RawMessage, pushHandler func(bchain.NotificationTyp // Initialize initializes PolisRPC instance. func (b *PolisRPC) Initialize() error { - chainName, err := b.GetChainInfoAndInitializeMempool(b) + ci, err := b.GetChainInfo() if err != nil { return err } + chainName := ci.Chain params := GetChainParams(chainName) From add504b57ed4a0a33b029b1493222232fad052d4 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 8 Apr 2019 14:39:29 +0200 Subject: [PATCH 31/39] Make ethereum type mempool parameters configurable --- bchain/coins/eth/ethrpc.go | 15 ++++++++------ bchain/mempool_ethereum_type.go | 23 ++++++++++++++++----- configs/coins/ethereum-classic.json | 17 ++++++++------- configs/coins/ethereum.json | 15 ++++++++------ configs/coins/ethereum_testnet_ropsten.json | 15 ++++++++------ 5 files changed, 55 insertions(+), 30 deletions(-) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 9656e6c3..eb0bdc2d 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -31,11 +31,13 @@ const ( // Configuration represents json config file type Configuration struct { - CoinName string `json:"coin_name"` - CoinShortcut string `json:"coin_shortcut"` - RPCURL string `json:"rpc_url"` - RPCTimeout int `json:"rpc_timeout"` - BlockAddressesToKeep int `json:"block_addresses_to_keep"` + CoinName string `json:"coin_name"` + CoinShortcut string `json:"coin_shortcut"` + RPCURL string `json:"rpc_url"` + RPCTimeout int `json:"rpc_timeout"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` + MempoolTxTimeoutHours int `json:"mempoolTxTimeoutHours"` + QueryBackendOnMempoolResync bool `json:"queryBackendOnMempoolResync"` } // EthereumRPC is an interface to JSON-RPC eth service. @@ -162,7 +164,8 @@ func (b *EthereumRPC) Initialize() error { // CreateMempool creates mempool if not already created, however does not initialize it func (b *EthereumRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) { if b.Mempool == nil { - b.Mempool = bchain.NewMempoolEthereumType(chain) + b.Mempool = bchain.NewMempoolEthereumType(chain, b.ChainConfig.MempoolTxTimeoutHours, b.ChainConfig.QueryBackendOnMempoolResync) + glog.Info("mempool created, MempoolTxTimeoutHours=", b.ChainConfig.MempoolTxTimeoutHours, ", QueryBackendOnMempoolResync=", b.ChainConfig.QueryBackendOnMempoolResync) } return b.Mempool, nil } diff --git a/bchain/mempool_ethereum_type.go b/bchain/mempool_ethereum_type.go index 4eb5b969..33e44e72 100644 --- a/bchain/mempool_ethereum_type.go +++ b/bchain/mempool_ethereum_type.go @@ -6,24 +6,28 @@ import ( "github.com/golang/glog" ) -const mempoolTimeoutTime = 24 * time.Hour const mempoolTimeoutRunPeriod = 10 * time.Minute // MempoolEthereumType is mempool handle of EthereumType chains type MempoolEthereumType struct { BaseMempool - nextTimeoutRun time.Time + mempoolTimeoutTime time.Duration + queryBackendOnResync bool + nextTimeoutRun time.Time } // NewMempoolEthereumType creates new mempool handler. -func NewMempoolEthereumType(chain BlockChain) *MempoolEthereumType { +func NewMempoolEthereumType(chain BlockChain, mempoolTxTimeoutHours int, queryBackendOnResync bool) *MempoolEthereumType { + mempoolTimeoutTime := time.Duration(mempoolTxTimeoutHours) * time.Hour return &MempoolEthereumType{ BaseMempool: BaseMempool{ chain: chain, txEntries: make(map[string]txEntry), addrDescToTx: make(map[string][]Outpoint), }, - nextTimeoutRun: time.Now().Add(mempoolTimeoutTime), + mempoolTimeoutTime: mempoolTimeoutTime, + queryBackendOnResync: queryBackendOnResync, + nextTimeoutRun: time.Now().Add(mempoolTimeoutTime), } } @@ -90,11 +94,20 @@ func (m *MempoolEthereumType) createTxEntry(txid string, txTime uint32) (txEntry // Resync ethereum type removes timed out transactions and returns number of transactions in mempool. // Transactions are added/removed by AddTransactionToMempool/RemoveTransactionFromMempool methods func (m *MempoolEthereumType) Resync() (int, error) { + if m.queryBackendOnResync { + txs, err := m.chain.GetMempoolTransactions() + if err != nil { + return 0, err + } + for _, txid := range txs { + m.AddTransactionToMempool(txid) + } + } m.mux.Lock() entries := len(m.txEntries) now := time.Now() if m.nextTimeoutRun.Before(now) { - threshold := now.Add(-mempoolTimeoutTime) + threshold := now.Add(-m.mempoolTimeoutTime) for txid, entry := range m.txEntries { if time.Unix(int64(entry.time), 0).Before(threshold) { m.removeEntryFromMempool(txid, entry) diff --git a/configs/coins/ethereum-classic.json b/configs/coins/ethereum-classic.json index 93e4268f..7889e415 100644 --- a/configs/coins/ethereum-classic.json +++ b/configs/coins/ethereum-classic.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Ethereum Classic", - "shortcut": "ETC", - "label": "Ethereum Classic", - "alias": "ethereum-classic" + "name": "Ethereum Classic", + "shortcut": "ETC", + "label": "Ethereum Classic", + "alias": "ethereum-classic" }, "ports": { "backend_rpc": 8037, @@ -41,17 +41,20 @@ "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", "explorer_url": "", - "additional_params": "-resyncindexperiod=4441", + "additional_params": "-resyncindexperiod=4441 -resyncmempoolperiod=2011", "block_chain": { "parse": true, "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, - "additional_params": {} + "additional_params": { + "mempoolTxTimeoutHours": 48, + "queryBackendOnMempoolResync": true + } } }, "meta": { "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/ethereum.json b/configs/coins/ethereum.json index 75ca8012..ebf40a4a 100644 --- a/configs/coins/ethereum.json +++ b/configs/coins/ethereum.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Ethereum", - "shortcut": "ETH", - "label": "Ethereum", - "alias": "ethereum" + "name": "Ethereum", + "shortcut": "ETH", + "label": "Ethereum", + "alias": "ethereum" }, "ports": { "backend_rpc": 8036, @@ -49,11 +49,14 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, - "additional_params": {} + "additional_params": { + "mempoolTxTimeoutHours": 48, + "queryBackendOnMempoolResync": false + } } }, "meta": { "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file diff --git a/configs/coins/ethereum_testnet_ropsten.json b/configs/coins/ethereum_testnet_ropsten.json index 1f8f603e..6c064490 100644 --- a/configs/coins/ethereum_testnet_ropsten.json +++ b/configs/coins/ethereum_testnet_ropsten.json @@ -1,9 +1,9 @@ { "coin": { - "name": "Ethereum Testnet Ropsten", - "shortcut": "tROP", - "label": "Ethereum Ropsten", - "alias": "ethereum_testnet_ropsten" + "name": "Ethereum Testnet Ropsten", + "shortcut": "tROP", + "label": "Ethereum Ropsten", + "alias": "ethereum_testnet_ropsten" }, "ports": { "backend_rpc": 18036, @@ -48,11 +48,14 @@ "mempool_workers": 8, "mempool_sub_workers": 2, "block_addresses_to_keep": 300, - "additional_params": {} + "additional_params": { + "mempoolTxTimeoutHours": 12, + "queryBackendOnMempoolResync": false + } } }, "meta": { "package_maintainer": "Petr Kracik", "package_maintainer_email": "petr.kracik@satoshilabs.com" } -} +} \ No newline at end of file From b367e25194c290c1d7a490590aa332622bcd0fba Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Mon, 8 Apr 2019 15:34:58 +0200 Subject: [PATCH 32/39] Fix bcash xpub derivation path #146 --- bchain/coins/bch/bcashparser.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bchain/coins/bch/bcashparser.go b/bchain/coins/bch/bcashparser.go index 8f6fd53b..d706af61 100644 --- a/bchain/coins/bch/bcashparser.go +++ b/bchain/coins/bch/bcashparser.go @@ -71,13 +71,7 @@ func NewBCashParser(params *chaincfg.Params, c *btc.Configuration) (*BCashParser return nil, fmt.Errorf("Unknown address format: %s", c.AddressFormat) } p := &BCashParser{ - BitcoinParser: &btc.BitcoinParser{ - BaseParser: &bchain.BaseParser{ - BlockAddressesToKeep: c.BlockAddressesToKeep, - AmountDecimalPoint: 8, - }, - Params: params, - }, + BitcoinParser: btc.NewBitcoinParser(params, c), AddressFormat: format, } p.OutputScriptToAddressesFunc = p.outputScriptToAddresses From 9f8cf071ddb8a4eeb514948c4024d0c01e39ddaf Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Wed, 10 Apr 2019 13:54:20 +0200 Subject: [PATCH 33/39] Stop displaying first seen tx time when unknown --- static/templates/txdetail.html | 2 +- static/templates/txdetail_ethereumtype.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/static/templates/txdetail.html b/static/templates/txdetail.html index 6e7327c8..6f8ae69c 100644 --- a/static/templates/txdetail.html +++ b/static/templates/txdetail.html @@ -4,7 +4,7 @@
{{$tx.Txid}}
-
{{if $tx.Confirmations}}mined{{else}}first seen{{end}} {{formatUnixTime $tx.Blocktime}}
+ {{- if $tx.Blocktime}}
{{if $tx.Confirmations}}mined{{else}}first seen{{end}} {{formatUnixTime $tx.Blocktime}}
{{end -}}
diff --git a/static/templates/txdetail_ethereumtype.html b/static/templates/txdetail_ethereumtype.html index 78cde7bd..8f873453 100644 --- a/static/templates/txdetail_ethereumtype.html +++ b/static/templates/txdetail_ethereumtype.html @@ -5,7 +5,7 @@ {{$tx.Txid}} {{if eq $tx.EthereumSpecific.Status 1}} ✔{{end}}{{if eq $tx.EthereumSpecific.Status 0}} ✘{{end}}
-
{{if $tx.Confirmations}}mined{{else}}first seen{{end}} {{formatUnixTime $tx.Blocktime}}
+ {{- if $tx.Blocktime}}
{{if $tx.Confirmations}}mined{{else}}first seen{{end}} {{formatUnixTime $tx.Blocktime}}
{{end -}}
From cd0202891bd671f21754e0d72628e0007a7a25c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Wed, 10 Apr 2019 10:35:50 +0200 Subject: [PATCH 34/39] Bump Dash backend version to 0.13.3.0 --- configs/coins/dash.json | 6 +++--- configs/coins/dash_testnet.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/coins/dash.json b/configs/coins/dash.json index b0198330..c4e4581a 100644 --- a/configs/coins/dash.json +++ b/configs/coins/dash.json @@ -22,10 +22,10 @@ "package_name": "backend-dash", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.13.2.0", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/dashcore-0.13.2.0-x86_64-linux-gnu.tar.gz", + "version": "0.13.3.0", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.3.0/dashcore-0.13.3.0-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.3.0/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" diff --git a/configs/coins/dash_testnet.json b/configs/coins/dash_testnet.json index 0d6952ca..974ea71e 100644 --- a/configs/coins/dash_testnet.json +++ b/configs/coins/dash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-dash-testnet", "package_revision": "satoshilabs-1", "system_user": "dash", - "version": "0.13.2.0", - "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/dashcore-0.13.2.0-x86_64-linux-gnu.tar.gz", + "version": "0.13.3.0", + "binary_url": "https://github.com/dashpay/dash/releases/download/v0.13.3.0/dashcore-0.13.3.0-x86_64-linux-gnu.tar.gz", "verification_type": "gpg-sha256", - "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.2.0/SHA256SUMS.asc", + "verification_source": "https://github.com/dashpay/dash/releases/download/v0.13.3.0/SHA256SUMS.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/dash-qt" From 8b38d3b7ea45b566f531ee9a4b429ccf3d6de40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Wed, 10 Apr 2019 10:43:23 +0200 Subject: [PATCH 35/39] Bump Ethereum to 1.8.25 --- configs/coins/ethereum.json | 6 +++--- configs/coins/ethereum_testnet_ropsten.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/coins/ethereum.json b/configs/coins/ethereum.json index ebf40a4a..fbf18f1a 100644 --- a/configs/coins/ethereum.json +++ b/configs/coins/ethereum.json @@ -21,10 +21,10 @@ "package_name": "backend-ethereum", "package_revision": "satoshilabs-1", "system_user": "ethereum", - "version": "1.8.23-c9427004", - "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.23-c9427004.tar.gz", + "version": "1.8.25-14ae1246", + "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.25-14ae1246.tar.gz", "verification_type": "gpg", - "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.23-c9427004.tar.gz.asc", + "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.25-14ae1246.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/geth --ipcdisable --syncmode full --cache 1024 --nat none --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port 38336 --ws --wsaddr 0.0.0.0 --wsport {{.Ports.BackendRPC}} --wsorigins \"*\" --rpc --rpcport 8136 -rpcaddr 0.0.0.0 --rpccorsdomain \"*\" --rpcvhosts \"*\" 2>>{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", diff --git a/configs/coins/ethereum_testnet_ropsten.json b/configs/coins/ethereum_testnet_ropsten.json index 6c064490..e55985de 100644 --- a/configs/coins/ethereum_testnet_ropsten.json +++ b/configs/coins/ethereum_testnet_ropsten.json @@ -20,10 +20,10 @@ "package_name": "backend-ethereum-testnet-ropsten", "package_revision": "satoshilabs-1", "system_user": "ethereum", - "version": "1.8.23-c9427004", - "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.23-c9427004.tar.gz", + "version": "1.8.25-14ae1246", + "binary_url": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.25-14ae1246.tar.gz", "verification_type": "gpg", - "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.23-c9427004.tar.gz.asc", + "verification_source": "https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-1.8.25-14ae1246.tar.gz.asc", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [], "exec_command_template": "/bin/sh -c '{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/geth --testnet --syncmode full --ipcdisable --cache 1024 --nat none --datadir {{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend --port 48336 --ws --wsaddr 0.0.0.0 --wsport {{.Ports.BackendRPC}} --wsorigins \"*\" 2>>{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/{{.Coin.Alias}}.log'", From 8c1691be8db430dde91e05294c90871194d4ac6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Krac=C3=ADk?= Date: Thu, 11 Apr 2019 14:40:28 +0200 Subject: [PATCH 36/39] Bump Bcash to 0.19.3 --- configs/coins/bcash.json | 6 +++--- configs/coins/bcash_testnet.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/coins/bcash.json b/configs/coins/bcash.json index 6c50aec0..5bc080df 100644 --- a/configs/coins/bcash.json +++ b/configs/coins/bcash.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.19.2", - "binary_url": "https://download.bitcoinabc.org/0.19.2/linux/bitcoin-abc-0.19.2-x86_64-linux-gnu.tar.gz", + "version": "0.19.3", + "binary_url": "https://download.bitcoinabc.org/0.19.3/linux/bitcoin-abc-0.19.3-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "615063ef2049f756c2cbb0b0823964a5eed5c5dd5420d10049a0b30563f631d2", + "verification_source": "4d784ea2e603e2dbf8d74f70263127a1fc1489a30d2ead60c0fc521d9e444076", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" diff --git a/configs/coins/bcash_testnet.json b/configs/coins/bcash_testnet.json index 1c6fee74..af05e7eb 100644 --- a/configs/coins/bcash_testnet.json +++ b/configs/coins/bcash_testnet.json @@ -22,10 +22,10 @@ "package_name": "backend-bcash-testnet", "package_revision": "satoshilabs-1", "system_user": "bcash", - "version": "0.19.2", - "binary_url": "https://download.bitcoinabc.org/0.19.2/linux/bitcoin-abc-0.19.2-x86_64-linux-gnu.tar.gz", + "version": "0.19.3", + "binary_url": "https://download.bitcoinabc.org/0.19.3/linux/bitcoin-abc-0.19.3-x86_64-linux-gnu.tar.gz", "verification_type": "sha256", - "verification_source": "615063ef2049f756c2cbb0b0823964a5eed5c5dd5420d10049a0b30563f631d2", + "verification_source": "4d784ea2e603e2dbf8d74f70263127a1fc1489a30d2ead60c0fc521d9e444076", "extract_command": "tar -C backend --strip 1 -xf", "exclude_files": [ "bin/bitcoin-qt" From 230b5e5d32759cc60c88696cee1615a29a6c46de Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Thu, 11 Apr 2019 14:57:39 +0200 Subject: [PATCH 37/39] Fix synchronization issue in mempool --- bchain/basemempool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bchain/basemempool.go b/bchain/basemempool.go index 68c09331..ae585bcc 100644 --- a/bchain/basemempool.go +++ b/bchain/basemempool.go @@ -65,6 +65,7 @@ func (a MempoolTxidEntries) Less(i, j int) bool { return hi > hj } +// removeEntryFromMempool removes entry from mempool structs. The caller is responsible for locking! func (m *BaseMempool) removeEntryFromMempool(txid string, entry txEntry) { delete(m.txEntries, txid) for _, si := range entry.addrIndexes { @@ -87,9 +88,9 @@ func (m *BaseMempool) removeEntryFromMempool(txid string, entry txEntry) { // GetAllEntries returns all mempool entries sorted by fist seen time in descending order func (m *BaseMempool) GetAllEntries() MempoolTxidEntries { - entries := make(MempoolTxidEntries, len(m.txEntries)) i := 0 m.mux.Lock() + entries := make(MempoolTxidEntries, len(m.txEntries)) for txid, entry := range m.txEntries { entries[i] = MempoolTxidEntry{ Txid: txid, From 8fb4772331ed895034f66e1ca2255f4bd0981103 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 12 Apr 2019 15:44:11 +0200 Subject: [PATCH 38/39] Swich from Fatal log to Error log in Blockbook main --- blockbook.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/blockbook.go b/blockbook.go index 4630b263..a0893ec9 100644 --- a/blockbook.go +++ b/blockbook.go @@ -119,34 +119,40 @@ func main() { if *repair { if err := db.RepairRocksDB(*dbPath); err != nil { - glog.Fatalf("RepairRocksDB %s: %v", *dbPath, err) + glog.Errorf("RepairRocksDB %s: %v", *dbPath, err) + return } return } if *blockchain == "" { - glog.Fatal("Missing blockchaincfg configuration parameter") + glog.Error("Missing blockchaincfg configuration parameter") + return } coin, coinShortcut, coinLabel, err := coins.GetCoinNameFromConfig(*blockchain) if err != nil { - glog.Fatal("config: ", err) + glog.Error("config: ", err) + return } // gspt.SetProcTitle("blockbook-" + normalizeName(coin)) metrics, err = common.GetMetrics(coin) if err != nil { - glog.Fatal("metrics: ", err) + glog.Error("metrics: ", err) + return } if chain, mempool, err = getBlockChainWithRetry(coin, *blockchain, pushSynchronizationHandler, metrics, 60); err != nil { - glog.Fatal("rpc: ", err) + glog.Error("rpc: ", err) + return } index, err = db.NewRocksDB(*dbPath, *dbCache, *dbMaxOpenFiles, chain.GetChainParser(), metrics) if err != nil { - glog.Fatal("rocksDB: ", err) + glog.Error("rocksDB: ", err) + return } defer index.Close() @@ -176,14 +182,16 @@ func main() { syncWorker, err = db.NewSyncWorker(index, chain, *syncWorkers, *syncChunk, *blockFrom, *dryRun, chanOsSignal, metrics, internalState) if err != nil { - glog.Fatalf("NewSyncWorker %v", err) + glog.Errorf("NewSyncWorker %v", err) + return } // set the DbState to open at this moment, after all important workers are initialized internalState.DbState = common.DbStateOpen err = index.StoreInternalState(internalState) if err != nil { - glog.Fatal("internalState: ", err) + glog.Error("internalState: ", err) + return } if *rollbackHeight >= 0 { From dffcded306036801e752d7c00489e944d5f79313 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 12 Apr 2019 16:46:54 +0200 Subject: [PATCH 39/39] Enable parallel sync only for initial sync --- db/sync.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/db/sync.go b/db/sync.go index e755c027..e81cbb2c 100644 --- a/db/sync.go +++ b/db/sync.go @@ -63,11 +63,15 @@ func (w *SyncWorker) ResyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync b if err == nil { w.is.FinishedSync(bh) } - return nil + return err case errSynced: // this is not actually error but flag that resync wasn't necessary w.is.FinishedSyncNoChange() w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) + if initialSync { + d := time.Since(start) + glog.Info("resync: finished in ", d) + } return nil } @@ -113,7 +117,8 @@ func (w *SyncWorker) resyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync b } // if parallel operation is enabled and the number of blocks to be connected is large, // use parallel routine to load majority of blocks - if w.syncWorkers > 1 { + // use parallel sync only in case of initial sync because it puts the db to inconsistent state + if w.syncWorkers > 1 && initialSync { remoteBestHeight, err := w.chain.GetBestBlockHeight() if err != nil { return err