diff --git a/Gopkg.lock b/Gopkg.lock index cf32c2f0..975e36b9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,8 +46,8 @@ [[projects]] name = "github.com/ethereum/go-ethereum" packages = [".","common","common/hexutil","common/math","core/types","crypto","crypto/secp256k1","crypto/sha3","ethclient","ethdb","log","metrics","p2p/netutil","params","rlp","rpc","trie"] - revision = "89451f7c382ad2185987ee369f16416f89c28a7d" - version = "v1.8.15" + revision = "8bbe72075e4e16442c4e28d999edee12e294329e" + version = "v1.8.17" [[projects]] name = "github.com/go-stack/stack" diff --git a/api/types.go b/api/types.go index c925be62..349cbfcc 100644 --- a/api/types.go +++ b/api/types.go @@ -4,10 +4,27 @@ import ( "blockbook/bchain" "blockbook/common" "blockbook/db" + "encoding/json" "math/big" "time" ) +// GetAddressOption specifies what data returns GetAddress api call +type GetAddressOption int + +const ( + // Basic - only that address is indexed and some basic info + Basic GetAddressOption = iota + // Balance - only balances + Balance + // TxidHistory - balances and txids, subject to paging + TxidHistory + // TxHistoryLight - balances and easily obtained tx data (not requiring request to backend), subject to paging + TxHistoryLight + // TxHistory - balances and full tx data, subject to paging + TxHistory +) + // APIError extends error by information if the error details should be returned to the end user type APIError struct { Text string @@ -26,97 +43,169 @@ func NewAPIError(s string, public bool) error { } } -// ScriptSig contains input script -type ScriptSig struct { - Hex string `json:"hex"` - Asm string `json:"asm,omitempty"` +// Amount is datatype holding amounts +type Amount big.Int + +// MarshalJSON Amount serialization +func (a *Amount) MarshalJSON() (out []byte, err error) { + if a == nil { + return []byte(`"0"`), nil + } + return []byte(`"` + (*big.Int)(a).String() + `"`), nil +} + +func (a *Amount) String() string { + if a == nil { + return "" + } + return (*big.Int)(a).String() +} + +// DecimalString returns amount with decimal point placed according to parameter d +func (a *Amount) DecimalString(d int) string { + return bchain.AmountToDecimalString((*big.Int)(a), d) +} + +// AsBigInt returns big.Int type for the Amount (empty if Amount is nil) +func (a *Amount) AsBigInt() big.Int { + if a == nil { + return *new(big.Int) + } + return big.Int(*a) } // Vin contains information about single transaction input type Vin struct { - Txid string `json:"txid"` - Vout uint32 `json:"vout"` + Txid string `json:"txid,omitempty"` + Vout uint32 `json:"vout,omitempty"` Sequence int64 `json:"sequence,omitempty"` N int `json:"n"` - ScriptSig ScriptSig `json:"scriptSig"` AddrDesc bchain.AddressDescriptor `json:"-"` Addresses []string `json:"addresses"` Searchable bool `json:"-"` - Value string `json:"value"` - ValueSat big.Int `json:"-"` -} - -// ScriptPubKey contains output script and addresses derived from it -type ScriptPubKey struct { - Hex string `json:"hex"` + ValueSat *Amount `json:"value,omitempty"` + Hex string `json:"hex,omitempty"` Asm string `json:"asm,omitempty"` - AddrDesc bchain.AddressDescriptor `json:"-"` - Addresses []string `json:"addresses"` - Searchable bool `json:"-"` - Type string `json:"type,omitempty"` } // Vout contains information about single transaction output type Vout struct { - Value string `json:"value"` - ValueSat big.Int `json:"-"` - N int `json:"n"` - ScriptPubKey ScriptPubKey `json:"scriptPubKey"` - Spent bool `json:"spent"` - SpentTxID string `json:"spentTxId,omitempty"` - SpentIndex int `json:"spentIndex,omitempty"` - SpentHeight int `json:"spentHeight,omitempty"` + ValueSat *Amount `json:"value,omitempty"` + N int `json:"n"` + Spent bool `json:"spent,omitempty"` + SpentTxID string `json:"spentTxId,omitempty"` + SpentIndex int `json:"spentIndex,omitempty"` + SpentHeight int `json:"spentHeight,omitempty"` + Hex string `json:"hex,omitempty"` + Asm string `json:"asm,omitempty"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Type string `json:"type,omitempty"` +} + +// Erc20Token contains info about ERC20 token held by an address +type Erc20Token struct { + Contract string `json:"contract"` + Transfers int `json:"transfers"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals int `json:"decimals"` + BalanceSat *Amount `json:"balance,omitempty"` + ContractIndex string `json:"-"` +} + +// TokenTransfer contains info about a token transfer done in a transaction +type TokenTransfer struct { + Type string `json:"type"` + From string `json:"from"` + To string `json:"to"` + Token string `json:"token"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals int `json:"decimals"` + Value *Amount `json:"value"` +} + +// EthereumSpecific contains ethereum specific transaction data +type EthereumSpecific struct { + Status int `json:"status"` // 1 OK, 0 Fail, -1 pending + Nonce uint64 `json:"nonce"` + GasLimit *big.Int `json:"gaslimit"` + GasUsed *big.Int `json:"gasused"` + GasPrice *Amount `json:"gasprice"` } // Tx holds information about a transaction type Tx struct { - Txid string `json:"txid"` - Version int32 `json:"version,omitempty"` - Locktime uint32 `json:"locktime,omitempty"` - Vin []Vin `json:"vin"` - Vout []Vout `json:"vout"` - Blockhash string `json:"blockhash,omitempty"` - Blockheight int `json:"blockheight"` - Confirmations uint32 `json:"confirmations"` - Time int64 `json:"time,omitempty"` - Blocktime int64 `json:"blocktime"` - ValueOut string `json:"valueOut"` - ValueOutSat big.Int `json:"-"` - Size int `json:"size,omitempty"` - ValueIn string `json:"valueIn"` - ValueInSat big.Int `json:"-"` - Fees string `json:"fees"` - FeesSat big.Int `json:"-"` - Hex string `json:"hex"` + Txid string `json:"txid"` + Version int32 `json:"version,omitempty"` + Locktime uint32 `json:"locktime,omitempty"` + Vin []Vin `json:"vin"` + Vout []Vout `json:"vout"` + Blockhash string `json:"blockhash,omitempty"` + Blockheight int `json:"blockheight"` + Confirmations uint32 `json:"confirmations"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime"` + Size int `json:"size,omitempty"` + ValueOutSat *Amount `json:"value"` + ValueInSat *Amount `json:"valueIn,omitempty"` + FeesSat *Amount `json:"fees,omitempty"` + Hex string `json:"hex,omitempty"` + CoinSpecificData interface{} `json:"-"` + CoinSpecificJSON json.RawMessage `json:"-"` + TokenTransfers []TokenTransfer `json:"tokentransfers,omitempty"` + EthereumSpecific *EthereumSpecific `json:"ethereumspecific,omitempty"` } // Paging contains information about paging for address, blocks and block type Paging struct { - Page int `json:"page"` - TotalPages int `json:"totalPages"` - ItemsOnPage int `json:"itemsOnPage"` + Page int `json:"page,omitempty"` + TotalPages int `json:"totalPages,omitempty"` + ItemsOnPage int `json:"itemsOnPage,omitempty"` +} + +const ( + // AddressFilterVoutOff disables filtering of transactions by vout + AddressFilterVoutOff = -1 + // AddressFilterVoutInputs specifies that only txs where the address is as input are returned + AddressFilterVoutInputs = -2 + // AddressFilterVoutOutputs specifies that only txs where the address is as output are returned + AddressFilterVoutOutputs = -3 +) + +// AddressFilter is used to filter data returned from GetAddress api method +type AddressFilter struct { + Vout int + Contract string + FromHeight uint32 + ToHeight uint32 } // Address holds information about address and its transactions type Address struct { Paging - AddrStr string `json:"addrStr"` - Balance string `json:"balance"` - TotalReceived string `json:"totalReceived"` - TotalSent string `json:"totalSent"` - UnconfirmedBalance string `json:"unconfirmedBalance"` - UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` - TxApperances int `json:"txApperances"` - Transactions []*Tx `json:"txs,omitempty"` - Txids []string `json:"transactions,omitempty"` + AddrStr string `json:"addrStr"` + BalanceSat *Amount `json:"balance"` + TotalReceivedSat *Amount `json:"totalReceived,omitempty"` + TotalSentSat *Amount `json:"totalSent,omitempty"` + UnconfirmedBalanceSat *Amount `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*Tx `json:"transactions,omitempty"` + Txids []string `json:"txids,omitempty"` + Nonce string `json:"nonce,omitempty"` + Erc20Contract *bchain.Erc20Contract `json:"erc20contract,omitempty"` + Erc20Tokens []Erc20Token `json:"erc20tokens,omitempty"` + Filter string `json:"-"` } // AddressUtxo holds information about address and its transactions type AddressUtxo struct { Txid string `json:"txid"` - Vout uint32 `json:"vout"` - Amount string `json:"amount"` - AmountSat big.Int `json:"satoshis"` + Vout int32 `json:"vout"` + AmountSat *Amount `json:"value"` Height int `json:"height,omitempty"` Confirmations int `json:"confirmations"` } @@ -150,6 +239,7 @@ type BlockbookInfo struct { InSyncMempool bool `json:"inSyncMempool"` LastMempoolTime time.Time `json:"lastMempoolTime"` MempoolSize int `json:"mempoolSize"` + Decimals int `json:"decimals"` DbSize int64 `json:"dbSize"` DbSizeFromColumns int64 `json:"dbSizeFromColumns,omitempty"` DbColumns []common.InternalStateColumn `json:"dbColumns,omitempty"` diff --git a/api/types_test.go b/api/types_test.go new file mode 100644 index 00000000..37026685 --- /dev/null +++ b/api/types_test.go @@ -0,0 +1,51 @@ +// build unittest + +package api + +import ( + "encoding/json" + "math/big" + "reflect" + "testing" +) + +func TestAmount_MarshalJSON(t *testing.T) { + type amounts struct { + A1 Amount `json:"a1"` + A2 Amount `json:"a2,omitempty"` + PA1 *Amount `json:"pa1"` + PA2 *Amount `json:"pa2,omitempty"` + } + tests := []struct { + name string + a amounts + want string + }{ + { + name: "empty", + want: `{"a1":"0","a2":"0","pa1":null}`, + }, + { + name: "1", + a: amounts{ + A1: (Amount)(*big.NewInt(123456)), + A2: (Amount)(*big.NewInt(787901)), + PA1: (*Amount)(big.NewInt(234567)), + PA2: (*Amount)(big.NewInt(890123)), + }, + want: `{"a1":"123456","a2":"787901","pa1":"234567","pa2":"890123"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := json.Marshal(&tt.a) + if err != nil { + t.Errorf("json.Marshal() error = %v", err) + return + } + if !reflect.DeepEqual(string(b), tt.want) { + t.Errorf("json.Marshal() = %v, want %v", string(b), tt.want) + } + }) + } +} diff --git a/api/typesv1.go b/api/typesv1.go new file mode 100644 index 00000000..301f5724 --- /dev/null +++ b/api/typesv1.go @@ -0,0 +1,220 @@ +package api + +import ( + "blockbook/bchain" + "math/big" +) + +// ScriptSigV1 is used for legacy api v1 +type ScriptSigV1 struct { + Hex string `json:"hex,omitempty"` + Asm string `json:"asm,omitempty"` +} + +// VinV1 is used for legacy api v1 +type VinV1 struct { + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Sequence int64 `json:"sequence,omitempty"` + N int `json:"n"` + ScriptSig ScriptSigV1 `json:"scriptSig"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Value string `json:"value"` + ValueSat big.Int `json:"-"` +} + +// ScriptPubKeyV1 is used for legacy api v1 +type ScriptPubKeyV1 struct { + Hex string `json:"hex,omitempty"` + Asm string `json:"asm,omitempty"` + AddrDesc bchain.AddressDescriptor `json:"-"` + Addresses []string `json:"addresses"` + Searchable bool `json:"-"` + Type string `json:"type,omitempty"` +} + +// VoutV1 is used for legacy api v1 +type VoutV1 struct { + Value string `json:"value"` + ValueSat big.Int `json:"-"` + N int `json:"n"` + ScriptPubKey ScriptPubKeyV1 `json:"scriptPubKey"` + Spent bool `json:"spent"` + SpentTxID string `json:"spentTxId,omitempty"` + SpentIndex int `json:"spentIndex,omitempty"` + SpentHeight int `json:"spentHeight,omitempty"` +} + +// TxV1 is used for legacy api v1 +type TxV1 struct { + Txid string `json:"txid"` + Version int32 `json:"version,omitempty"` + Locktime uint32 `json:"locktime,omitempty"` + Vin []VinV1 `json:"vin"` + Vout []VoutV1 `json:"vout"` + Blockhash string `json:"blockhash,omitempty"` + Blockheight int `json:"blockheight"` + Confirmations uint32 `json:"confirmations"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime"` + ValueOut string `json:"valueOut"` + ValueOutSat big.Int `json:"-"` + Size int `json:"size,omitempty"` + ValueIn string `json:"valueIn"` + ValueInSat big.Int `json:"-"` + Fees string `json:"fees"` + FeesSat big.Int `json:"-"` + Hex string `json:"hex"` +} + +// AddressV1 is used for legacy api v1 +type AddressV1 struct { + Paging + AddrStr string `json:"addrStr"` + Balance string `json:"balance"` + TotalReceived string `json:"totalReceived"` + TotalSent string `json:"totalSent"` + UnconfirmedBalance string `json:"unconfirmedBalance"` + UnconfirmedTxApperances int `json:"unconfirmedTxApperances"` + TxApperances int `json:"txApperances"` + Transactions []*TxV1 `json:"txs,omitempty"` + Txids []string `json:"transactions,omitempty"` +} + +// AddressUtxoV1 is used for legacy api v1 +type AddressUtxoV1 struct { + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Amount string `json:"amount"` + AmountSat big.Int `json:"satoshis"` + Height int `json:"height,omitempty"` + Confirmations int `json:"confirmations"` +} + +// BlockV1 contains information about block +type BlockV1 struct { + Paging + bchain.BlockInfo + TxCount int `json:"TxCount"` + Transactions []*TxV1 `json:"txs,omitempty"` +} + +// TxToV1 converts Tx to TxV1 +func (w *Worker) TxToV1(tx *Tx) *TxV1 { + d := w.chainParser.AmountDecimals() + vinV1 := make([]VinV1, len(tx.Vin)) + for i := range tx.Vin { + v := &tx.Vin[i] + vinV1[i] = VinV1{ + AddrDesc: v.AddrDesc, + Addresses: v.Addresses, + N: v.N, + ScriptSig: ScriptSigV1{ + Asm: v.Asm, + Hex: v.Hex, + }, + Searchable: v.Searchable, + Sequence: v.Sequence, + Txid: v.Txid, + Value: v.ValueSat.DecimalString(d), + ValueSat: v.ValueSat.AsBigInt(), + Vout: v.Vout, + } + } + voutV1 := make([]VoutV1, len(tx.Vout)) + for i := range tx.Vout { + v := &tx.Vout[i] + voutV1[i] = VoutV1{ + N: v.N, + ScriptPubKey: ScriptPubKeyV1{ + AddrDesc: v.AddrDesc, + Addresses: v.Addresses, + Asm: v.Asm, + Hex: v.Hex, + Searchable: v.Searchable, + Type: v.Type, + }, + Spent: v.Spent, + SpentHeight: v.SpentHeight, + SpentIndex: v.SpentIndex, + SpentTxID: v.SpentTxID, + Value: v.ValueSat.DecimalString(d), + ValueSat: v.ValueSat.AsBigInt(), + } + } + return &TxV1{ + Blockhash: tx.Blockhash, + Blockheight: tx.Blockheight, + Blocktime: tx.Blocktime, + Confirmations: tx.Confirmations, + Fees: tx.FeesSat.DecimalString(d), + FeesSat: tx.FeesSat.AsBigInt(), + Hex: tx.Hex, + Locktime: tx.Locktime, + Size: tx.Size, + Time: tx.Time, + Txid: tx.Txid, + ValueIn: tx.ValueInSat.DecimalString(d), + ValueInSat: tx.ValueInSat.AsBigInt(), + ValueOut: tx.ValueOutSat.DecimalString(d), + ValueOutSat: tx.ValueOutSat.AsBigInt(), + Version: tx.Version, + Vin: vinV1, + Vout: voutV1, + } +} + +func (w *Worker) transactionsToV1(txs []*Tx) []*TxV1 { + v1 := make([]*TxV1, len(txs)) + for i := range txs { + v1[i] = w.TxToV1(txs[i]) + } + return v1 +} + +// AddressToV1 converts Address to AddressV1 +func (w *Worker) AddressToV1(a *Address) *AddressV1 { + d := w.chainParser.AmountDecimals() + return &AddressV1{ + AddrStr: a.AddrStr, + Balance: a.BalanceSat.DecimalString(d), + Paging: a.Paging, + TotalReceived: a.TotalReceivedSat.DecimalString(d), + TotalSent: a.TotalSentSat.DecimalString(d), + Transactions: w.transactionsToV1(a.Transactions), + TxApperances: a.TxApperances, + Txids: a.Txids, + UnconfirmedBalance: a.UnconfirmedBalanceSat.DecimalString(d), + UnconfirmedTxApperances: a.UnconfirmedTxApperances, + } +} + +// AddressUtxoToV1 converts []AddressUtxo to []AddressUtxoV1 +func (w *Worker) AddressUtxoToV1(au []AddressUtxo) []AddressUtxoV1 { + d := w.chainParser.AmountDecimals() + v1 := make([]AddressUtxoV1, len(au)) + for i := range au { + utxo := &au[i] + v1[i] = AddressUtxoV1{ + AmountSat: utxo.AmountSat.AsBigInt(), + Amount: utxo.AmountSat.DecimalString(d), + Confirmations: utxo.Confirmations, + Height: utxo.Height, + Txid: utxo.Txid, + Vout: uint32(utxo.Vout), + } + } + return v1 +} + +// BlockToV1 converts Address to Address1 +func (w *Worker) BlockToV1(b *Block) *BlockV1 { + return &BlockV1{ + BlockInfo: b.BlockInfo, + Paging: b.Paging, + Transactions: w.transactionsToV1(b.Transactions), + TxCount: b.TxCount, + } +} diff --git a/api/worker.go b/api/worker.go index 8b14639c..cb91a49b 100644 --- a/api/worker.go +++ b/api/worker.go @@ -2,9 +2,11 @@ package api import ( "blockbook/bchain" + "blockbook/bchain/coins/eth" "blockbook/common" "blockbook/db" "bytes" + "encoding/json" "fmt" "math/big" "strconv" @@ -20,6 +22,7 @@ type Worker struct { txCache *db.TxCache chain bchain.BlockChain chainParser bchain.BlockChainParser + chainType bchain.ChainType is *common.InternalState } @@ -30,6 +33,7 @@ func NewWorker(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, is txCache: txCache, chain: chain, chainParser: chain.GetChainParser(), + chainType: chain.GetChainParser().GetChainType(), is: is, } return w, nil @@ -47,7 +51,7 @@ func (w *Worker) getAddressesFromVout(vout *bchain.Vout) (bchain.AddressDescript // setSpendingTxToVout is helper function, that finds transaction that spent given output and sets it to the output // there is no direct index for the operation, it must be found using addresses -> txaddresses -> tx func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) error { - err := w.db.GetAddrDescTransactions(vout.ScriptPubKey.AddrDesc, height, ^uint32(0), func(t string, index uint32, isOutput bool) error { + err := w.db.GetAddrDescTransactions(vout.AddrDesc, height, ^uint32(0), func(t string, index int32, isOutput bool) error { if isOutput == false { tsp, err := w.db.GetTxAddresses(t) if err != nil { @@ -55,7 +59,7 @@ func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) err } else if tsp == nil { glog.Warning("DB inconsistency: tx ", t, ": not found in txAddresses") } else if len(tsp.Inputs) > int(index) { - if tsp.Inputs[index].ValueSat.Cmp(&vout.ValueSat) == 0 { + if tsp.Inputs[index].ValueSat.Cmp((*big.Int)(vout.ValueSat)) == 0 { spentTx, spentHeight, err := w.txCache.GetTransaction(t) if err != nil { glog.Warning("Tx ", t, ": not found") @@ -80,7 +84,7 @@ func (w *Worker) setSpendingTxToVout(vout *Vout, txid string, height uint32) err // GetSpendingTxid returns transaction id of transaction that spent given output func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) { start := time.Now() - tx, err := w.GetTransaction(txid, false) + tx, err := w.GetTransaction(txid, false, false) if err != nil { return "", err } @@ -96,18 +100,27 @@ func (w *Worker) GetSpendingTxid(txid string, n int) (string, error) { } // GetTransaction reads transaction data from txid -func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) { - start := time.Now() +func (w *Worker) GetTransaction(txid string, spendingTxs bool, specificJSON bool) (*Tx, error) { bchainTx, height, err := w.txCache.GetTransaction(txid) if err != nil { return nil, NewAPIError(fmt.Sprintf("Tx not found, %v", err), true) } + return w.GetTransactionFromBchainTx(bchainTx, height, spendingTxs, specificJSON) +} + +// GetTransactionFromBchainTx reads transaction data from txid +func (w *Worker) GetTransactionFromBchainTx(bchainTx *bchain.Tx, height uint32, spendingTxs bool, specificJSON bool) (*Tx, error) { + var err error var ta *db.TxAddresses + var tokens []TokenTransfer + var ethSpecific *EthereumSpecific var blockhash string if bchainTx.Confirmations > 0 { - ta, err = w.db.GetTxAddresses(txid) - if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + if w.chainType == bchain.ChainBitcoinType { + ta, err = w.db.GetTxAddresses(bchainTx.Txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainTx.Txid) + } } blockhash, err = w.db.GetBlockHash(height) if err != nil { @@ -115,6 +128,7 @@ func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) { } } var valInSat, valOutSat, feesSat big.Int + var pValInSat *big.Int vins := make([]Vin, len(bchainTx.Vin)) for i := range bchainTx.Vin { bchainVin := &bchainTx.Vin[i] @@ -123,46 +137,55 @@ func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) { vin.N = i vin.Vout = bchainVin.Vout vin.Sequence = int64(bchainVin.Sequence) - vin.ScriptSig.Hex = bchainVin.ScriptSig.Hex - // bchainVin.Txid=="" is coinbase transaction - if bchainVin.Txid != "" { - // load spending addresses from TxAddresses - tas, err := w.db.GetTxAddresses(bchainVin.Txid) - if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) - } - if tas == nil { - // mempool transactions are not in TxAddresses but confirmed should be there, log a problem - if bchainTx.Confirmations > 0 { - glog.Warning("DB inconsistency: tx ", bchainVin.Txid, ": not found in txAddresses") - } - // try to load from backend - otx, _, err := w.txCache.GetTransaction(bchainVin.Txid) + vin.Hex = bchainVin.ScriptSig.Hex + if w.chainType == bchain.ChainBitcoinType { + // bchainVin.Txid=="" is coinbase transaction + if bchainVin.Txid != "" { + // load spending addresses from TxAddresses + tas, err := w.db.GetTxAddresses(bchainVin.Txid) if err != nil { - return nil, errors.Annotatef(err, "txCache.GetTransaction %v", bchainVin.Txid) + return nil, errors.Annotatef(err, "GetTxAddresses %v", bchainVin.Txid) } - if len(otx.Vout) > int(vin.Vout) { - vout := &otx.Vout[vin.Vout] - vin.ValueSat = vout.ValueSat - vin.AddrDesc, vin.Addresses, vin.Searchable, err = w.getAddressesFromVout(vout) + if tas == nil { + // mempool transactions are not in TxAddresses but confirmed should be there, log a problem + if bchainTx.Confirmations > 0 { + glog.Warning("DB inconsistency: tx ", bchainVin.Txid, ": not found in txAddresses") + } + // try to load from backend + otx, _, err := w.txCache.GetTransaction(bchainVin.Txid) if err != nil { - glog.Errorf("getAddressesFromVout error %v, vout %+v", err, vout) - } - } - } else { - if len(tas.Outputs) > int(vin.Vout) { - output := &tas.Outputs[vin.Vout] - vin.ValueSat = output.ValueSat - vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) - vin.AddrDesc = output.AddrDesc - vin.Addresses, vin.Searchable, err = output.Addresses(w.chainParser) - if err != nil { - glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) + return nil, errors.Annotatef(err, "txCache.GetTransaction %v", bchainVin.Txid) + } + if len(otx.Vout) > int(vin.Vout) { + vout := &otx.Vout[vin.Vout] + vin.ValueSat = (*Amount)(&vout.ValueSat) + vin.AddrDesc, vin.Addresses, vin.Searchable, err = w.getAddressesFromVout(vout) + if err != nil { + glog.Errorf("getAddressesFromVout error %v, vout %+v", err, vout) + } + } + } else { + if len(tas.Outputs) > int(vin.Vout) { + output := &tas.Outputs[vin.Vout] + vin.ValueSat = (*Amount)(&output.ValueSat) + vin.AddrDesc = output.AddrDesc + vin.Addresses, vin.Searchable, err = output.Addresses(w.chainParser) + if err != nil { + glog.Errorf("output.Addresses error %v, tx %v, output %v", err, bchainVin.Txid, i) + } } } + valInSat.Add(&valInSat, (*big.Int)(vin.ValueSat)) + } + } else if w.chainType == bchain.ChainEthereumType { + if len(bchainVin.Addresses) > 0 { + vin.AddrDesc, err = w.chainParser.GetAddrDescFromAddress(bchainVin.Addresses[0]) + if err != nil { + glog.Errorf("GetAddrDescFromAddress error %v, tx %v, bchainVin %v", err, bchainTx.Txid, bchainVin) + } + vin.Addresses = bchainVin.Addresses + vin.Searchable = true } - vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) - valInSat.Add(&valInSat, &vin.ValueSat) } } vouts := make([]Vout, len(bchainTx.Vout)) @@ -170,11 +193,10 @@ func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) { bchainVout := &bchainTx.Vout[i] vout := &vouts[i] vout.N = i - vout.ValueSat = bchainVout.ValueSat - vout.Value = w.chainParser.AmountToDecimalString(&bchainVout.ValueSat) + vout.ValueSat = (*Amount)(&bchainVout.ValueSat) valOutSat.Add(&valOutSat, &bchainVout.ValueSat) - vout.ScriptPubKey.Hex = bchainVout.ScriptPubKey.Hex - vout.ScriptPubKey.AddrDesc, vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = w.getAddressesFromVout(bchainVout) + vout.Hex = bchainVout.ScriptPubKey.Hex + vout.AddrDesc, vout.Addresses, vout.Searchable, err = w.getAddressesFromVout(bchainVout) if err != nil { glog.V(2).Infof("getAddressesFromVout error %v, %v, output %v", err, bchainTx.Txid, bchainVout.N) } @@ -183,60 +205,132 @@ func (w *Worker) GetTransaction(txid string, spendingTxs bool) (*Tx, error) { if spendingTxs && vout.Spent { err = w.setSpendingTxToVout(vout, bchainTx.Txid, height) if err != nil { - glog.Errorf("setSpendingTxToVout error %v, %v, output %v", err, vout.ScriptPubKey.AddrDesc, vout.N) + glog.Errorf("setSpendingTxToVout error %v, %v, output %v", err, vout.AddrDesc, vout.N) } } } } - // for coinbase transactions valIn is 0 - feesSat.Sub(&valInSat, &valOutSat) - if feesSat.Sign() == -1 { - feesSat.SetUint64(0) + if w.chainType == bchain.ChainBitcoinType { + // for coinbase transactions valIn is 0 + feesSat.Sub(&valInSat, &valOutSat) + if feesSat.Sign() == -1 { + feesSat.SetUint64(0) + } + pValInSat = &valInSat + } else if w.chainType == bchain.ChainEthereumType { + ets, err := w.chainParser.EthereumTypeGetErc20FromTx(bchainTx) + if err != nil { + glog.Errorf("GetErc20FromTx error %v, %v", err, bchainTx) + } + tokens = make([]TokenTransfer, len(ets)) + for i := range ets { + e := &ets[i] + cd, err := w.chainParser.GetAddrDescFromAddress(e.Contract) + if err != nil { + glog.Errorf("GetAddrDescFromAddress error %v, contract %v", err, e.Contract) + continue + } + erc20c, err := w.chain.EthereumTypeGetErc20ContractInfo(cd) + if err != nil { + glog.Errorf("GetErc20ContractInfo error %v, contract %v", err, e.Contract) + } + if erc20c == nil { + erc20c = &bchain.Erc20Contract{Name: e.Contract} + } + tokens[i] = TokenTransfer{ + Type: "ERC20", + Token: e.Contract, + From: e.From, + To: e.To, + Decimals: erc20c.Decimals, + Value: (*Amount)(&e.Tokens), + Name: erc20c.Name, + Symbol: erc20c.Symbol, + } + } + ethTxData := eth.GetEthereumTxData(bchainTx) + // mempool txs do not have fees yet + if ethTxData.GasUsed != nil { + feesSat.Mul(ethTxData.GasPrice, ethTxData.GasUsed) + } + if len(bchainTx.Vout) > 0 { + valOutSat = bchainTx.Vout[0].ValueSat + } + ethSpecific = &EthereumSpecific{ + GasLimit: ethTxData.GasLimit, + GasPrice: (*Amount)(ethTxData.GasPrice), + GasUsed: ethTxData.GasUsed, + Nonce: ethTxData.Nonce, + Status: ethTxData.Status, + } } // for now do not return size, we would have to compute vsize of segwit transactions // size:=len(bchainTx.Hex) / 2 - r := &Tx{ - Blockhash: blockhash, - Blockheight: int(height), - Blocktime: bchainTx.Blocktime, - Confirmations: bchainTx.Confirmations, - Fees: w.chainParser.AmountToDecimalString(&feesSat), - FeesSat: feesSat, - Locktime: bchainTx.LockTime, - Time: bchainTx.Time, - Txid: bchainTx.Txid, - ValueIn: w.chainParser.AmountToDecimalString(&valInSat), - ValueInSat: valInSat, - ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), - ValueOutSat: valOutSat, - Version: bchainTx.Version, - Hex: bchainTx.Hex, - Vin: vins, - Vout: vouts, + var sj json.RawMessage + if specificJSON { + sj, err = w.chain.GetTransactionSpecific(bchainTx) + if err != nil { + return nil, err + } } - if spendingTxs { - glog.Info("GetTransaction ", txid, " finished in ", time.Since(start)) + r := &Tx{ + Blockhash: blockhash, + Blockheight: int(height), + Blocktime: bchainTx.Blocktime, + Confirmations: bchainTx.Confirmations, + FeesSat: (*Amount)(&feesSat), + Locktime: bchainTx.LockTime, + Time: bchainTx.Time, + Txid: bchainTx.Txid, + ValueInSat: (*Amount)(pValInSat), + ValueOutSat: (*Amount)(&valOutSat), + Version: bchainTx.Version, + Hex: bchainTx.Hex, + Vin: vins, + Vout: vouts, + CoinSpecificData: bchainTx.CoinSpecificData, + CoinSpecificJSON: sj, + TokenTransfers: tokens, + EthereumSpecific: ethSpecific, } return r, nil } -func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool) ([]string, error) { +func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool, filter *AddressFilter) ([]string, error) { var err error txids := make([]string, 0, 4) - if !mempool { - err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { + addFilteredTxid := func(txid string, vout int32, isOutput bool) error { + if filter.Vout == AddressFilterVoutOff || + (filter.Vout == AddressFilterVoutInputs && !isOutput) || + (filter.Vout == AddressFilterVoutOutputs && isOutput) || + (vout == int32(filter.Vout)) { txids = append(txids, txid) - return nil - }) + } + return nil + } + if mempool { + o, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) if err != nil { return nil, err } + for _, m := range o { + vout := m.Vout + isOutput := true + if vout < 0 { + isOutput = false + vout = ^vout + } + addFilteredTxid(m.Txid, vout, isOutput) + } } else { - m, err := w.chain.GetMempoolTransactionsForAddrDesc(addrDesc) + to := filter.ToHeight + if to == 0 { + to = ^uint32(0) + } + err = w.db.GetAddrDescTransactions(addrDesc, filter.FromHeight, to, addFilteredTxid) if err != nil { return nil, err } - txids = append(txids, m...) } return txids, nil } @@ -244,8 +338,8 @@ func (w *Worker) getAddressTxids(addrDesc bchain.AddressDescriptor, mempool bool func (t *Tx) getAddrVoutValue(addrDesc bchain.AddressDescriptor) *big.Int { var val big.Int for _, vout := range t.Vout { - if bytes.Equal(vout.ScriptPubKey.AddrDesc, addrDesc) { - val.Add(&val, &vout.ValueSat) + if bytes.Equal(vout.AddrDesc, addrDesc) && vout.ValueSat != nil { + val.Add(&val, (*big.Int)(vout.ValueSat)) } } return &val @@ -254,8 +348,8 @@ func (t *Tx) getAddrVoutValue(addrDesc bchain.AddressDescriptor) *big.Int { func (t *Tx) getAddrVinValue(addrDesc bchain.AddressDescriptor) *big.Int { var val big.Int for _, vin := range t.Vin { - if bytes.Equal(vin.AddrDesc, addrDesc) { - val.Add(&val, &vin.ValueSat) + if bytes.Equal(vin.AddrDesc, addrDesc) && vin.ValueSat != nil { + val.Add(&val, (*big.Int)(vin.ValueSat)) } } return &val @@ -285,9 +379,8 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn tai := &ta.Inputs[i] vin := &vins[i] vin.N = i - vin.ValueSat = tai.ValueSat - vin.Value = w.chainParser.AmountToDecimalString(&vin.ValueSat) - valInSat.Add(&valInSat, &vin.ValueSat) + vin.ValueSat = (*Amount)(&tai.ValueSat) + valInSat.Add(&valInSat, &tai.ValueSat) vin.Addresses, vin.Searchable, err = tai.Addresses(w.chainParser) if err != nil { glog.Errorf("tai.Addresses error %v, tx %v, input %v, tai %+v", err, txid, i, tai) @@ -298,10 +391,9 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn tao := &ta.Outputs[i] vout := &vouts[i] vout.N = i - vout.ValueSat = tao.ValueSat - vout.Value = w.chainParser.AmountToDecimalString(&vout.ValueSat) - valOutSat.Add(&valOutSat, &vout.ValueSat) - vout.ScriptPubKey.Addresses, vout.ScriptPubKey.Searchable, err = tao.Addresses(w.chainParser) + vout.ValueSat = (*Amount)(&tao.ValueSat) + valOutSat.Add(&valOutSat, &tao.ValueSat) + vout.Addresses, vout.Searchable, err = tao.Addresses(w.chainParser) if err != nil { glog.Errorf("tai.Addresses error %v, tx %v, output %v, tao %+v", err, txid, i, tao) } @@ -317,11 +409,11 @@ func (w *Worker) txFromTxAddress(txid string, ta *db.TxAddresses, bi *db.BlockIn Blockheight: int(ta.Height), Blocktime: bi.Time, Confirmations: bestheight - ta.Height + 1, - Fees: w.chainParser.AmountToDecimalString(&feesSat), + FeesSat: (*Amount)(&feesSat), Time: bi.Time, Txid: txid, - ValueIn: w.chainParser.AmountToDecimalString(&valInSat), - ValueOut: w.chainParser.AmountToDecimalString(&valOutSat), + ValueInSat: (*Amount)(&valInSat), + ValueOutSat: (*Amount)(&valOutSat), Vin: vins, Vout: vouts, } @@ -349,8 +441,94 @@ func computePaging(count, page, itemsOnPage int) (Paging, int, int, int) { }, from, to, page } +func (w *Worker) getEthereumTypeAddressBalances(addrDesc bchain.AddressDescriptor, option GetAddressOption, filter *AddressFilter) (*db.AddrBalance, []Erc20Token, *bchain.Erc20Contract, uint64, error) { + var ( + ba *db.AddrBalance + erc20t []Erc20Token + ci *bchain.Erc20Contract + n uint64 + ) + ca, err := w.db.GetAddrDescContracts(addrDesc) + if err != nil { + return nil, nil, nil, 0, NewAPIError(fmt.Sprintf("Address not found, %v", err), true) + } + if ca != nil { + ba = &db.AddrBalance{ + Txs: uint32(ca.EthTxs), + } + var b *big.Int + b, err = w.chain.EthereumTypeGetBalance(addrDesc) + if err != nil { + return nil, nil, nil, 0, errors.Annotatef(err, "EthereumTypeGetBalance %v", addrDesc) + } + if b != nil { + ba.BalanceSat = *b + } + n, err = w.chain.EthereumTypeGetNonce(addrDesc) + if err != nil { + return nil, nil, nil, 0, errors.Annotatef(err, "EthereumTypeGetNonce %v", addrDesc) + } + var filterDesc bchain.AddressDescriptor + if filter.Contract != "" { + filterDesc, err = w.chainParser.GetAddrDescFromAddress(filter.Contract) + if err != nil { + return nil, nil, nil, 0, NewAPIError(fmt.Sprintf("Invalid contract filter, %v", err), true) + } + } + erc20t = make([]Erc20Token, len(ca.Contracts)) + var j int + for i, c := range ca.Contracts { + if len(filterDesc) > 0 { + if !bytes.Equal(filterDesc, c.Contract) { + continue + } + // filter only transactions by this contract + filter.Vout = i + 1 + } + ci, err := w.chain.EthereumTypeGetErc20ContractInfo(c.Contract) + if err != nil { + return nil, nil, nil, 0, errors.Annotatef(err, "EthereumTypeGetErc20ContractInfo %v", c.Contract) + } + if ci == nil { + ci = &bchain.Erc20Contract{} + addresses, _, _ := w.chainParser.GetAddressesFromAddrDesc(c.Contract) + if len(addresses) > 0 { + ci.Contract = addresses[0] + ci.Name = addresses[0] + } + } + // do not read contract balances etc in case of Basic option + if option != Basic { + b, err = w.chain.EthereumTypeGetErc20ContractBalance(addrDesc, c.Contract) + if err != nil { + // return nil, nil, nil, errors.Annotatef(err, "EthereumTypeGetErc20ContractBalance %v %v", addrDesc, c.Contract) + glog.Warningf("EthereumTypeGetErc20ContractBalance addr %v, contract %v, %v", addrDesc, c.Contract, err) + } + } else { + b = nil + } + erc20t[j] = Erc20Token{ + BalanceSat: (*Amount)(b), + Contract: ci.Contract, + Name: ci.Name, + Symbol: ci.Symbol, + Transfers: int(c.Txs), + Decimals: ci.Decimals, + ContractIndex: strconv.Itoa(i + 1), + } + j++ + } + erc20t = erc20t[:j] + ci, err = w.chain.EthereumTypeGetErc20ContractInfo(addrDesc) + if err != nil { + return nil, nil, nil, 0, err + } + } + return ba, erc20t, ci, n, nil +} + // GetAddress computes address value and gets transactions for given address -func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids bool) (*Address, error) { +func (w *Worker) GetAddress(address string, page int, txsOnPage int, option GetAddressOption, filter *AddressFilter) (*Address, error) { start := time.Now() page-- if page < 0 { @@ -360,118 +538,148 @@ func (w *Worker) GetAddress(address string, page int, txsOnPage int, onlyTxids b if err != nil { return nil, NewAPIError(fmt.Sprintf("Invalid address, %v", err), true) } - // ba can be nil if the address is only in mempool! - ba, err := w.db.GetAddrDescBalance(addrDesc) - if err != nil { - return nil, NewAPIError(fmt.Sprintf("Address not found, %v", err), true) - } - // convert the address to the format defined by the parser - addresses, _, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) - if err != nil { - glog.V(2).Infof("GetAddressesFromAddrDesc error %v, %v", err, addrDesc) - } - if len(addresses) == 1 { - address = addresses[0] - } - txc, err := w.getAddressTxids(addrDesc, false) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v false", address) - } - txc = UniqueTxidsInReverse(txc) - var txm []string - // if there are only unconfirmed transactions, ba is nil - if ba == nil { - ba = &db.AddrBalance{} - page = 0 - } - txm, err = w.getAddressTxids(addrDesc, true) - if err != nil { - return nil, errors.Annotatef(err, "getAddressTxids %v true", address) - } - txm = UniqueTxidsInReverse(txm) - // check if the address exist - if len(txc)+len(txm) == 0 { - return &Address{ - AddrStr: address, - }, nil - } - bestheight, _, err := w.db.GetBestBlock() - if err != nil { - return nil, errors.Annotatef(err, "GetBestBlock") - } - pg, from, to, page := computePaging(len(txc), page, txsOnPage) - var txs []*Tx - var txids []string - if onlyTxids { - txids = make([]string, len(txm)+to-from) - } else { - txs = make([]*Tx, len(txm)+to-from) - } - txi := 0 - // load mempool transactions - var uBalSat big.Int - for _, tx := range txm { - tx, err := w.GetTransaction(tx, false) - // mempool transaction may fail + var ( + ba *db.AddrBalance + erc20t []Erc20Token + erc20c *bchain.Erc20Contract + txm []string + txs []*Tx + txids []string + pg Paging + uBalSat big.Int + totalReceived, totalSent *big.Int + nonce string + ) + if w.chainType == bchain.ChainEthereumType { + var n uint64 + ba, erc20t, erc20c, n, err = w.getEthereumTypeAddressBalances(addrDesc, option, filter) if err != nil { - glog.Error("GetTransaction in mempool ", tx, ": ", err) - } else { - uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) - uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) - if page == 0 { - if onlyTxids { - txids[txi] = tx.Txid + return nil, err + } + nonce = strconv.Itoa(int(n)) + } else { + // ba can be nil if the address is only in mempool! + ba, err = w.db.GetAddrDescBalance(addrDesc) + if err != nil { + return nil, NewAPIError(fmt.Sprintf("Address not found, %v", err), true) + } + } + // get tx history if requested by option or check mempool if there are some transactions for a new address + if option >= TxidHistory || ba == nil { + // convert the address to the format defined by the parser + addresses, _, err := w.chainParser.GetAddressesFromAddrDesc(addrDesc) + if err != nil { + glog.V(2).Infof("GetAddressesFromAddrDesc error %v, %v", err, addrDesc) + } + if len(addresses) == 1 { + address = addresses[0] + } + txm, err = w.getAddressTxids(addrDesc, true, filter) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v true", addrDesc) + } + txm = UniqueTxidsInReverse(txm) + // if there are only unconfirmed transactions, there is no paging + if ba == nil { + ba = &db.AddrBalance{} + page = 0 + } + if option >= TxidHistory { + txc, err := w.getAddressTxids(addrDesc, false, filter) + if err != nil { + return nil, errors.Annotatef(err, "getAddressTxids %v false", addrDesc) + } + txc = UniqueTxidsInReverse(txc) + bestheight, _, err := w.db.GetBestBlock() + if err != nil { + return nil, errors.Annotatef(err, "GetBestBlock") + } + var from, to int + pg, from, to, page = computePaging(len(txc), page, txsOnPage) + if option == TxidHistory { + txids = make([]string, len(txm)+to-from) + } else { + txs = make([]*Tx, len(txm)+to-from) + } + txi := 0 + // get mempool transactions + for _, txid := range txm { + tx, err := w.GetTransaction(txid, false, false) + // mempool transaction may fail + if err != nil { + glog.Error("GetTransaction in mempool ", tx, ": ", err) } else { - txs[txi] = tx + uBalSat.Add(&uBalSat, tx.getAddrVoutValue(addrDesc)) + uBalSat.Sub(&uBalSat, tx.getAddrVinValue(addrDesc)) + if page == 0 { + if option == TxidHistory { + txids[txi] = tx.Txid + } else { + txs[txi] = tx + } + txi++ + } + } + } + // get confirmed transactions + for i := from; i < to; i++ { + txid := txc[i] + if option == TxidHistory { + txids[txi] = txid + } else { + // only ChainBitcoinType supports TxHistoryLight + if option == TxHistoryLight && w.chainType == bchain.ChainBitcoinType { + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + bi, err := w.db.GetBlockInfo(ta.Height) + if err != nil { + return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) + } + if bi == nil { + glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) + } else { + txs[txi], err = w.GetTransaction(txid, false, true) + if err != nil { + return nil, errors.Annotatef(err, "GetTransaction %v", txid) + } + } } txi++ } + if option == TxidHistory { + txids = txids[:txi] + } else if option >= TxHistoryLight { + txs = txs[:txi] + } } } - if len(txc) != int(ba.Txs) { - glog.Warning("DB inconsistency for address ", address, ": number of txs from column addresses ", len(txc), ", from addressBalance ", ba.Txs) - } - for i := from; i < to; i++ { - txid := txc[i] - if onlyTxids { - txids[txi] = txid - } else { - ta, err := w.db.GetTxAddresses(txid) - if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) - } - if ta == nil { - glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") - continue - } - bi, err := w.db.GetBlockInfo(ta.Height) - if err != nil { - return nil, errors.Annotatef(err, "GetBlockInfo %v", ta.Height) - } - if bi == nil { - glog.Warning("DB inconsistency: block height ", ta.Height, ": not found in db") - continue - } - txs[txi] = w.txFromTxAddress(txid, ta, bi, bestheight) - } - txi++ - } - if onlyTxids { - txids = txids[:txi] - } else { - txs = txs[:txi] + if w.chainType == bchain.ChainBitcoinType { + totalReceived = ba.ReceivedSat() + totalSent = &ba.SentSat } r := &Address{ Paging: pg, AddrStr: address, - Balance: w.chainParser.AmountToDecimalString(&ba.BalanceSat), - TotalReceived: w.chainParser.AmountToDecimalString(ba.ReceivedSat()), - TotalSent: w.chainParser.AmountToDecimalString(&ba.SentSat), - TxApperances: len(txc), - UnconfirmedBalance: w.chainParser.AmountToDecimalString(&uBalSat), + BalanceSat: (*Amount)(&ba.BalanceSat), + TotalReceivedSat: (*Amount)(totalReceived), + TotalSentSat: (*Amount)(totalSent), + TxApperances: int(ba.Txs), + UnconfirmedBalanceSat: (*Amount)(&uBalSat), UnconfirmedTxApperances: len(txm), Transactions: txs, Txids: txids, + Erc20Contract: erc20c, + Erc20Tokens: erc20t, + Nonce: nonce, } glog.Info("GetAddress ", address, " finished in ", time.Since(start)) return r, nil @@ -488,7 +696,7 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt r := make([]AddressUtxo, 0, 8) if !onlyConfirmed { // get utxo from mempool - txm, err := w.getAddressTxids(addrDesc, true) + txm, err := w.getAddressTxids(addrDesc, true, &AddressFilter{Vout: AddressFilterVoutOff}) if err != nil { return nil, errors.Annotatef(err, "getAddressTxids %v true", address) } @@ -520,9 +728,8 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt if !e { r = append(r, AddressUtxo{ Txid: bchainTx.Txid, - Vout: uint32(i), - AmountSat: vout.ValueSat, - Amount: w.chainParser.AmountToDecimalString(&vout.ValueSat), + Vout: int32(i), + AmountSat: (*Amount)(&vout.ValueSat), }) } } @@ -538,14 +745,10 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt var checksum big.Int // ba can be nil if the address is only in mempool! if ba != nil && ba.BalanceSat.Uint64() > 0 { - type outpoint struct { - txid string - vout uint32 - } - outpoints := make([]outpoint, 0, 8) - err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, vout uint32, isOutput bool) error { + outpoints := make([]bchain.Outpoint, 0, 8) + err = w.db.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, vout int32, isOutput bool) error { if isOutput { - outpoints = append(outpoints, outpoint{txid, vout}) + outpoints = append(outpoints, bchain.Outpoint{Txid: txid, Vout: vout}) } return nil }) @@ -562,29 +765,28 @@ func (w *Worker) GetAddressUtxo(address string, onlyConfirmed bool) ([]AddressUt bestheight := int(b) for i := len(outpoints) - 1; i >= 0 && checksum.Int64() > 0; i-- { o := outpoints[i] - if lastTxid != o.txid { - ta, err = w.db.GetTxAddresses(o.txid) + if lastTxid != o.Txid { + ta, err = w.db.GetTxAddresses(o.Txid) if err != nil { return nil, err } - lastTxid = o.txid + lastTxid = o.Txid } if ta == nil { - glog.Warning("DB inconsistency: tx ", o.txid, ": not found in txAddresses") + glog.Warning("DB inconsistency: tx ", o.Txid, ": not found in txAddresses") } else { - if len(ta.Outputs) <= int(o.vout) { - glog.Warning("DB inconsistency: txAddresses ", o.txid, " does not have enough outputs") + if len(ta.Outputs) <= int(o.Vout) { + glog.Warning("DB inconsistency: txAddresses ", o.Txid, " does not have enough outputs") } else { - if !ta.Outputs[o.vout].Spent { - v := ta.Outputs[o.vout].ValueSat + if !ta.Outputs[o.Vout].Spent { + v := ta.Outputs[o.Vout].ValueSat // report only outpoints that are not spent in mempool - _, e := spentInMempool[o.txid+strconv.Itoa(int(o.vout))] + _, e := spentInMempool[o.Txid+strconv.Itoa(int(o.Vout))] if !e { r = append(r, AddressUtxo{ - Txid: o.txid, - Vout: o.vout, - AmountSat: v, - Amount: w.chainParser.AmountToDecimalString(&v), + Txid: o.Txid, + Vout: o.Vout, + AmountSat: (*Amount)(&v), Height: int(ta.Height), Confirmations: bestheight - int(ta.Height) + 1, }) @@ -669,24 +871,37 @@ func (w *Worker) GetBlock(bid string, page int, txsOnPage int) (*Block, error) { return nil, errors.Annotatef(err, "GetBestBlock") } pg, from, to, page := computePaging(txCount, page, txsOnPage) - glog.Info("GetBlock ", bid, ", page ", page, " finished in ", time.Since(start)) txs := make([]*Tx, to-from) txi := 0 for i := from; i < to; i++ { txid := bi.Txids[i] - ta, err := w.db.GetTxAddresses(txid) - if err != nil { - return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + if w.chainType == bchain.ChainBitcoinType { + ta, err := w.db.GetTxAddresses(txid) + if err != nil { + return nil, errors.Annotatef(err, "GetTxAddresses %v", txid) + } + if ta == nil { + glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") + continue + } + txs[txi] = w.txFromTxAddress(txid, ta, dbi, bestheight) + } else { + txs[txi], err = w.GetTransaction(txid, false, false) + if err != nil { + return nil, err + } } - if ta == nil { - glog.Warning("DB inconsistency: tx ", txid, ": not found in txAddresses") - continue - } - txs[txi] = w.txFromTxAddress(txid, ta, dbi, bestheight) txi++ } + if bi.Prev == "" && bi.Height != 0 { + bi.Prev, _ = w.db.GetBlockHash(bi.Height - 1) + } + if bi.Next == "" && bi.Height != bestheight { + bi.Next, _ = w.db.GetBlockHash(bi.Height + 1) + } txs = txs[:txi] bi.Txids = nil + glog.Info("GetBlock ", bid, ", page ", page, " finished in ", time.Since(start)) return &Block{ Paging: pg, BlockInfo: *bi, @@ -725,6 +940,7 @@ func (w *Worker) GetSystemInfo(internal bool) (*SystemInfo, error) { InSyncMempool: ms, LastMempoolTime: mt, MempoolSize: msz, + Decimals: w.chainParser.AmountDecimals(), DbSize: w.db.DatabaseSizeOnDisk(), DbSizeFromColumns: dbs, DbColumns: dbc, diff --git a/bchain/basechain.go b/bchain/basechain.go new file mode 100644 index 00000000..1de31a48 --- /dev/null +++ b/bchain/basechain.go @@ -0,0 +1,55 @@ +package bchain + +import ( + "errors" + "math/big" +) + +// BaseChain is base type for bchain.BlockChain +type BaseChain struct { + Parser BlockChainParser + Testnet bool + Network string +} + +// TODO more bchain.BlockChain methods + +// GetChainParser returns BlockChainParser +func (b *BaseChain) GetChainParser() BlockChainParser { + return b.Parser +} + +// IsTestnet returns true if the blockchain is testnet +func (b *BaseChain) IsTestnet() bool { + return b.Testnet +} + +// GetNetworkName returns network name +func (b *BaseChain) GetNetworkName() string { + return b.Network +} + +// EthereumTypeGetBalance is not supported +func (b *BaseChain) EthereumTypeGetBalance(addrDesc AddressDescriptor) (*big.Int, error) { + return nil, errors.New("Not supported") +} + +// EthereumTypeGetNonce is not supported +func (b *BaseChain) EthereumTypeGetNonce(addrDesc AddressDescriptor) (uint64, error) { + return 0, errors.New("Not supported") +} + +// EthereumTypeEstimateGas is not supported +func (b *BaseChain) EthereumTypeEstimateGas(params map[string]interface{}) (uint64, error) { + return 0, errors.New("Not supported") +} + +// EthereumTypeGetErc20ContractInfo is not supported +func (b *BaseChain) EthereumTypeGetErc20ContractInfo(contractDesc AddressDescriptor) (*Erc20Contract, error) { + return nil, errors.New("Not supported") +} + +// EthereumTypeGetErc20ContractBalance is not supported +func (b *BaseChain) EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc AddressDescriptor) (*big.Int, error) { + return nil, errors.New("Not supported") +} diff --git a/bchain/baseparser.go b/bchain/baseparser.go index 48bb752e..3060b3c2 100644 --- a/bchain/baseparser.go +++ b/bchain/baseparser.go @@ -34,10 +34,14 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { var r big.Int s := string(n) i := strings.IndexByte(s, '.') + d := p.AmountDecimalPoint + if d > len(zeros) { + d = len(zeros) + } if i == -1 { - s = s + zeros[:p.AmountDecimalPoint] + s = s + zeros[:d] } else { - z := p.AmountDecimalPoint - len(s) + i + 1 + z := d - len(s) + i + 1 if z > 0 { s = s[:i] + s[i+1:] + zeros[:z] } else { @@ -50,18 +54,24 @@ func (p *BaseParser) AmountToBigInt(n json.Number) (big.Int, error) { return r, nil } -// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place -func (p *BaseParser) AmountToDecimalString(a *big.Int) string { +// AmountToDecimalString converts amount in big.Int to string with decimal point in the place defined by the parameter d +func AmountToDecimalString(a *big.Int, d int) string { + if a == nil { + return "" + } n := a.String() var s string if n[0] == '-' { n = n[1:] s = "-" } - if len(n) <= p.AmountDecimalPoint { - n = zeros[:p.AmountDecimalPoint-len(n)+1] + n + if d > len(zeros) { + d = len(zeros) } - i := len(n) - p.AmountDecimalPoint + if len(n) <= d { + n = zeros[:d-len(n)+1] + n + } + i := len(n) - d ad := strings.TrimRight(n[i:], "0") if len(ad) > 0 { n = n[:i] + "." + ad @@ -71,6 +81,16 @@ func (p *BaseParser) AmountToDecimalString(a *big.Int) string { return s + n } +// AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place +func (p *BaseParser) AmountToDecimalString(a *big.Int) string { + return AmountToDecimalString(a, p.AmountDecimalPoint) +} + +// AmountDecimals returns number of decimal places in amounts +func (p *BaseParser) AmountDecimals() int { + return p.AmountDecimalPoint +} + // ParseTxFromJson parses JSON message containing transaction and returns Tx struct func (p *BaseParser) ParseTxFromJson(msg json.RawMessage) (*Tx, error) { var tx Tx @@ -125,9 +145,9 @@ func (p *BaseParser) UnpackBlockHash(buf []byte) (string, error) { return hex.EncodeToString(buf), nil } -// IsUTXOChain returns true if the block chain is UTXO type, otherwise false -func (p *BaseParser) IsUTXOChain() bool { - return true +// GetChainType is type of the blockchain, default is ChainBitcoinType +func (p *BaseParser) GetChainType() ChainType { + return ChainBitcoinType } // PackTx packs transaction to byte array using protobuf @@ -171,6 +191,7 @@ func (p *BaseParser) PackTx(tx *Tx, height uint32, blockTime int64) ([]byte, err Locktime: tx.LockTime, Vin: pti, Vout: pto, + Version: tx.Version, } if pt.Hex, err = hex.DecodeString(tx.Hex); err != nil { return nil, errors.Annotatef(err, "Hex %v", tx.Hex) @@ -230,6 +251,12 @@ func (p *BaseParser) UnpackTx(buf []byte) (*Tx, uint32, error) { Txid: txid, Vin: vin, Vout: vout, + Version: pt.Version, } return &tx, pt.Height, nil } + +// EthereumTypeGetErc20FromTx is unsupported +func (p *BaseParser) EthereumTypeGetErc20FromTx(tx *Tx) ([]Erc20Transfer, error) { + return nil, errors.New("Not supported") +} diff --git a/bchain/baseparser_test.go b/bchain/baseparser_test.go index cfb20a68..668ed267 100644 --- a/bchain/baseparser_test.go +++ b/bchain/baseparser_test.go @@ -27,7 +27,8 @@ var amounts = []struct { {big.NewInt(-8), "-0.00000008", 8, "!"}, {big.NewInt(-89012345678), "-890.12345678", 8, "!"}, {big.NewInt(-12345), "-0.00012345", 8, "!"}, - {big.NewInt(12345678), "0.123456789012", 8, "0.12345678"}, // test of truncation of too many decimal places + {big.NewInt(12345678), "0.123456789012", 8, "0.12345678"}, // test of truncation of too many decimal places + {big.NewInt(12345678), "0.0000000000000000000000000000000012345678", 1234, "!"}, // test of too big number decimal places } func TestBaseParser_AmountToDecimalString(t *testing.T) { diff --git a/bchain/coins/blockchain.go b/bchain/coins/blockchain.go index 2e02db45..bf28ff8d 100644 --- a/bchain/coins/blockchain.go +++ b/bchain/coins/blockchain.go @@ -11,6 +11,7 @@ import ( "blockbook/bchain/coins/eth" "blockbook/bchain/coins/gamecredits" "blockbook/bchain/coins/grs" + "blockbook/bchain/coins/liquid" "blockbook/bchain/coins/litecoin" "blockbook/bchain/coins/monacoin" "blockbook/bchain/coins/myriad" @@ -57,6 +58,7 @@ func init() { BlockChainFactories["Monacoin Testnet"] = monacoin.NewMonacoinRPC BlockChainFactories["DigiByte"] = digibyte.NewDigiByteRPC BlockChainFactories["Myriad"] = myriad.NewMyriadRPC + BlockChainFactories["Liquid"] = liquid.NewLiquidRPC BlockChainFactories["Groestlcoin"] = grs.NewGroestlcoinRPC BlockChainFactories["Groestlcoin Testnet"] = grs.NewGroestlcoinRPC } @@ -187,9 +189,9 @@ func (c *blockChainWithMetrics) GetTransaction(txid string) (v *bchain.Tx, err e return c.b.GetTransaction(txid) } -func (c *blockChainWithMetrics) GetTransactionSpecific(txid string) (v json.RawMessage, err error) { +func (c *blockChainWithMetrics) GetTransactionSpecific(tx *bchain.Tx) (v json.RawMessage, err error) { defer func(s time.Time) { c.observeRPCLatency("GetTransactionSpecific", s, err) }(time.Now()) - return c.b.GetTransactionSpecific(txid) + return c.b.GetTransactionSpecific(tx) } func (c *blockChainWithMetrics) GetTransactionForMempool(txid string) (v *bchain.Tx, err error) { @@ -221,12 +223,12 @@ func (c *blockChainWithMetrics) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc return count, err } -func (c *blockChainWithMetrics) GetMempoolTransactions(address string) (v []string, err error) { +func (c *blockChainWithMetrics) GetMempoolTransactions(address string) (v []bchain.Outpoint, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactions", s, err) }(time.Now()) return c.b.GetMempoolTransactions(address) } -func (c *blockChainWithMetrics) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []string, err error) { +func (c *blockChainWithMetrics) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []bchain.Outpoint, err error) { defer func(s time.Time) { c.observeRPCLatency("GetMempoolTransactionsForAddrDesc", s, err) }(time.Now()) return c.b.GetMempoolTransactionsForAddrDesc(addrDesc) } @@ -239,3 +241,28 @@ func (c *blockChainWithMetrics) GetMempoolEntry(txid string) (v *bchain.MempoolE func (c *blockChainWithMetrics) GetChainParser() bchain.BlockChainParser { return c.b.GetChainParser() } + +func (c *blockChainWithMetrics) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) (v *big.Int, err error) { + defer func(s time.Time) { c.observeRPCLatency("EthereumTypeGetBalance", s, err) }(time.Now()) + return c.b.EthereumTypeGetBalance(addrDesc) +} + +func (c *blockChainWithMetrics) EthereumTypeGetNonce(addrDesc bchain.AddressDescriptor) (v uint64, err error) { + defer func(s time.Time) { c.observeRPCLatency("EthereumTypeGetNonce", s, err) }(time.Now()) + return c.b.EthereumTypeGetNonce(addrDesc) +} + +func (c *blockChainWithMetrics) EthereumTypeEstimateGas(params map[string]interface{}) (v uint64, err error) { + defer func(s time.Time) { c.observeRPCLatency("EthereumTypeEstimateGas", s, err) }(time.Now()) + return c.b.EthereumTypeEstimateGas(params) +} + +func (c *blockChainWithMetrics) EthereumTypeGetErc20ContractInfo(contractDesc bchain.AddressDescriptor) (v *bchain.Erc20Contract, err error) { + defer func(s time.Time) { c.observeRPCLatency("EthereumTypeGetErc20ContractInfo", s, err) }(time.Now()) + return c.b.EthereumTypeGetErc20ContractInfo(contractDesc) +} + +func (c *blockChainWithMetrics) EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc bchain.AddressDescriptor) (v *big.Int, err error) { + defer func(s time.Time) { c.observeRPCLatency("EthereumTypeGetErc20ContractInfo", s, err) }(time.Now()) + return c.b.EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc) +} diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index dfecbf71..b7b8f143 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -11,24 +11,22 @@ import ( "math/big" "net" "net/http" + "runtime/debug" "time" "github.com/btcsuite/btcd/wire" - "github.com/golang/glog" "github.com/juju/errors" ) // BitcoinRPC is an interface to JSON-RPC bitcoind service. type BitcoinRPC struct { + *bchain.BaseChain client http.Client rpcURL string user string password string - Parser bchain.BlockChainParser - Testnet bool - Network string - Mempool *bchain.UTXOMempool + Mempool *bchain.MempoolBitcoinType ParseBlocks bool pushHandler func(bchain.NotificationType) mq *bchain.MQ @@ -36,6 +34,7 @@ type BitcoinRPC struct { RPCMarshaler RPCMarshaler } +// Configuration represents json config file type Configuration struct { CoinName string `json:"coin_name"` CoinShortcut string `json:"coin_shortcut"` @@ -84,6 +83,7 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT } s := &BitcoinRPC{ + BaseChain: &bchain.BaseChain{}, client: http.Client{Timeout: time.Duration(c.RPCTimeout) * time.Second, Transport: transport}, rpcURL: c.RPCURL, user: c.RPCUser, @@ -115,7 +115,7 @@ func (b *BitcoinRPC) GetChainInfoAndInitializeMempool(bc bchain.BlockChain) (str } b.mq = mq - b.Mempool = bchain.NewUTXOMempool(bc, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers) + b.Mempool = bchain.NewMempoolBitcoinType(bc, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers) return chainName, nil } @@ -158,14 +158,6 @@ func (b *BitcoinRPC) Shutdown(ctx context.Context) error { return nil } -func (b *BitcoinRPC) IsTestnet() bool { - return b.Testnet -} - -func (b *BitcoinRPC) GetNetworkName() string { - return b.Network -} - func (b *BitcoinRPC) GetCoinName() string { return b.ChainConfig.CoinName } @@ -580,7 +572,7 @@ func (b *BitcoinRPC) GetBlockWithoutHeader(hash string, height uint32) (*bchain. return block, nil } -// GetBlockRaw returns block with given hash as bytes. +// GetBlockRaw returns block with given hash as bytes func (b *BitcoinRPC) GetBlockRaw(hash string) ([]byte, error) { glog.V(1).Info("rpc: getblock (verbosity=0) ", hash) @@ -602,7 +594,7 @@ func (b *BitcoinRPC) GetBlockRaw(hash string) ([]byte, error) { return hex.DecodeString(res.Result) } -// GetBlockFull returns block with given hash. +// GetBlockFull returns block with given hash func (b *BitcoinRPC) GetBlockFull(hash string) (*bchain.Block, error) { glog.V(1).Info("rpc: getblock (verbosity=2) ", hash) @@ -624,7 +616,7 @@ func (b *BitcoinRPC) GetBlockFull(hash string) (*bchain.Block, error) { return &res.Result, nil } -// GetMempool returns transactions in mempool. +// GetMempool returns transactions in mempool func (b *BitcoinRPC) GetMempool() ([]string, error) { glog.V(1).Info("rpc: getrawmempool") @@ -641,7 +633,7 @@ func (b *BitcoinRPC) GetMempool() ([]string, error) { return res.Result, nil } -// GetTransactionForMempool returns a transaction by the transaction ID. +// GetTransactionForMempool returns a transaction by the transaction ID // It could be optimized for mempool, i.e. without block time and confirmations func (b *BitcoinRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { glog.V(1).Info("rpc: getrawtransaction nonverbose ", txid) @@ -668,13 +660,14 @@ func (b *BitcoinRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { return tx, nil } -// GetTransaction returns a transaction by the transaction ID. +// GetTransaction returns a transaction by the transaction ID func (b *BitcoinRPC) GetTransaction(txid string) (*bchain.Tx, error) { - r, err := b.GetTransactionSpecific(txid) + r, err := b.getRawTransaction(txid) if err != nil { return nil, err } tx, err := b.Parser.ParseTxFromJson(r) + tx.CoinSpecificData = r if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } @@ -682,7 +675,15 @@ func (b *BitcoinRPC) GetTransaction(txid string) (*bchain.Tx, error) { } // GetTransactionSpecific returns json as returned by backend, with all coin specific data -func (b *BitcoinRPC) GetTransactionSpecific(txid string) (json.RawMessage, error) { +func (b *BitcoinRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, error) { + if csd, ok := tx.CoinSpecificData.(json.RawMessage); ok { + return csd, nil + } + return b.getRawTransaction(tx.Txid) +} + +// getRawTransaction returns json as returned by backend, with all coin specific data +func (b *BitcoinRPC) getRawTransaction(txid string) (json.RawMessage, error) { glog.V(1).Info("rpc: getrawtransaction ", txid) res := ResGetRawTransaction{} @@ -702,18 +703,18 @@ func (b *BitcoinRPC) GetTransactionSpecific(txid string) (json.RawMessage, error // ResyncMempool gets mempool transactions and maps output scripts to transactions. // ResyncMempool is not reentrant, it should be called from a single thread. -// It returns number of transactions in mempool +// Return value is number of transactions in mempool func (b *BitcoinRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { return b.Mempool.Resync(onNewTxAddr) } // GetMempoolTransactions returns slice of mempool transactions for given address -func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]string, error) { +func (b *BitcoinRPC) GetMempoolTransactions(address string) ([]bchain.Outpoint, error) { return b.Mempool.GetTransactions(address) } // GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor -func (b *BitcoinRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, error) { +func (b *BitcoinRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]bchain.Outpoint, error) { return b.Mempool.GetAddrDescTransactions(addrDesc) } @@ -828,6 +829,7 @@ func safeDecodeResponse(body io.ReadCloser, res interface{}) (err error) { defer func() { if r := recover(); r != nil { glog.Error("unmarshal json recovered from panic: ", r, "; data: ", string(data)) + debug.PrintStack() if len(data) > 0 && len(data) < 2048 { err = errors.Errorf("Error: %v", string(data)) } else { @@ -872,8 +874,3 @@ func (b *BitcoinRPC) Call(req interface{}, res interface{}) error { } return safeDecodeResponse(httpRes.Body, &res) } - -// GetChainParser returns BlockChainParser -func (b *BitcoinRPC) GetChainParser() bchain.BlockChainParser { - return b.Parser -} diff --git a/bchain/coins/eth/erc20.go b/bchain/coins/eth/erc20.go new file mode 100644 index 00000000..217a0d60 --- /dev/null +++ b/bchain/coins/eth/erc20.go @@ -0,0 +1,218 @@ +package eth + +import ( + "blockbook/bchain" + "context" + "encoding/hex" + "math/big" + "strings" + "sync" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/golang/glog" + "github.com/juju/errors" +) + +var erc20abi = `[{"constant":true,"inputs":[],"name":"name","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function","signature":"0x06fdde03"}, +{"constant":true,"inputs":[],"name":"symbol","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function","signature":"0x95d89b41"}, +{"constant":true,"inputs":[],"name":"decimals","outputs":[{"name":"","type":"uint8"}],"payable":false,"type":"function","signature":"0x313ce567"}, +{"constant":true,"inputs":[],"name":"totalSupply","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function","signature":"0x18160ddd"}, +{"constant":true,"inputs":[{"name":"_owner","type":"address"}],"name":"balanceOf","outputs":[{"name":"balance","type":"uint256"}],"payable":false,"type":"function","signature":"0x70a08231"}, +{"constant":false,"inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transfer","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function","signature":"0xa9059cbb"}, +{"constant":false,"inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"name":"transferFrom","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function","signature":"0x23b872dd"}, +{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"name":"approve","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function","signature":"0x095ea7b3"}, +{"constant":true,"inputs":[{"name":"_owner","type":"address"},{"name":"_spender","type":"address"}],"name":"allowance","outputs":[{"name":"remaining","type":"uint256"}],"payable":false,"type":"function","signature":"0xdd62ed3e"}, +{"anonymous":false,"inputs":[{"indexed":true,"name":"_from","type":"address"},{"indexed":true,"name":"_to","type":"address"},{"indexed":false,"name":"_value","type":"uint256"}],"name":"Transfer","type":"event","signature":"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"}, +{"anonymous":false,"inputs":[{"indexed":true,"name":"_owner","type":"address"},{"indexed":true,"name":"_spender","type":"address"},{"indexed":false,"name":"_value","type":"uint256"}],"name":"Approval","type":"event","signature":"0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925"}, +{"inputs":[{"name":"_initialAmount","type":"uint256"},{"name":"_tokenName","type":"string"},{"name":"_decimalUnits","type":"uint8"},{"name":"_tokenSymbol","type":"string"}],"payable":false,"type":"constructor"}, +{"constant":false,"inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"},{"name":"_extraData","type":"bytes"}],"name":"approveAndCall","outputs":[{"name":"success","type":"bool"}],"payable":false,"type":"function","signature":"0xcae9ca51"}, +{"constant":true,"inputs":[],"name":"version","outputs":[{"name":"","type":"string"}],"payable":false,"type":"function","signature":"0x54fd4d50"}]` + +// doing the parsing/processing without using go-ethereum/accounts/abi library, it is simple to get data from Transfer event +const erc20TransferMethodSignature = "0xa9059cbb" +const erc20TransferEventSignature = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" +const erc20NameSignature = "0x06fdde03" +const erc20SymbolSignature = "0x95d89b41" +const erc20DecimalsSignature = "0x313ce567" +const erc20BalanceOf = "0x70a08231" + +var cachedContracts = make(map[string]*bchain.Erc20Contract) +var cachedContractsMux sync.Mutex + +func addressFromPaddedHex(s string) (string, error) { + var t big.Int + var ok bool + if has0xPrefix(s) { + _, ok = t.SetString(s[2:], 16) + } else { + _, ok = t.SetString(s, 16) + } + if !ok { + return "", errors.New("Data is not a number") + } + a := ethcommon.BigToAddress(&t) + return a.String(), nil +} + +func erc20GetTransfersFromLog(logs []*rpcLog) ([]bchain.Erc20Transfer, error) { + var r []bchain.Erc20Transfer + for _, l := range logs { + if len(l.Topics) == 3 && l.Topics[0] == erc20TransferEventSignature { + var t big.Int + _, ok := t.SetString(l.Data, 0) + if !ok { + return nil, errors.New("Data is not a number") + } + from, err := addressFromPaddedHex(l.Topics[1]) + if err != nil { + return nil, err + } + to, err := addressFromPaddedHex(l.Topics[2]) + if err != nil { + return nil, err + } + r = append(r, bchain.Erc20Transfer{ + Contract: strings.ToLower(l.Address), + From: strings.ToLower(from), + To: strings.ToLower(to), + Tokens: t, + }) + } + } + return r, nil +} + +func erc20GetTransfersFromTx(tx *rpcTransaction) ([]bchain.Erc20Transfer, error) { + var r []bchain.Erc20Transfer + if len(tx.Payload) == 128+len(erc20TransferMethodSignature) && strings.HasPrefix(tx.Payload, erc20TransferMethodSignature) { + to, err := addressFromPaddedHex(tx.Payload[len(erc20TransferMethodSignature) : 64+len(erc20TransferMethodSignature)]) + if err != nil { + return nil, err + } + var t big.Int + _, ok := t.SetString(tx.Payload[len(erc20TransferMethodSignature)+64:], 16) + if !ok { + return nil, errors.New("Data is not a number") + } + r = append(r, bchain.Erc20Transfer{ + Contract: strings.ToLower(tx.To), + From: strings.ToLower(tx.From), + To: strings.ToLower(to), + Tokens: t, + }) + } + return r, nil +} + +func (b *EthereumRPC) ethCall(data, to string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + var r string + err := b.rpc.CallContext(ctx, &r, "eth_call", map[string]interface{}{ + "data": data, + "to": to, + }, "latest") + if err != nil { + return "", err + } + return r, nil +} + +func parseErc20NumericProperty(contractDesc bchain.AddressDescriptor, data string) *big.Int { + if has0xPrefix(data) { + data = data[2:] + } + if len(data) == 64 { + var n big.Int + _, ok := n.SetString(data, 16) + if ok { + return &n + } + } + glog.Warning("Cannot parse '", data, "' for contract ", contractDesc) + return nil +} + +func parseErc20StringProperty(contractDesc bchain.AddressDescriptor, data string) string { + if has0xPrefix(data) { + data = data[2:] + } + if len(data) > 128 { + n := parseErc20NumericProperty(contractDesc, data[64:128]) + if n != nil { + l := n.Uint64() + if 2*int(l) <= len(data)-128 { + b, err := hex.DecodeString(data[128 : 128+2*l]) + if err == nil { + return string(b) + } + } + } + } + glog.Warning("Cannot parse '", data, "' for contract ", contractDesc) + return "" +} + +// EthereumTypeGetErc20ContractInfo returns information about ERC20 contract +func (b *EthereumRPC) EthereumTypeGetErc20ContractInfo(contractDesc bchain.AddressDescriptor) (*bchain.Erc20Contract, error) { + cds := string(contractDesc) + cachedContractsMux.Lock() + contract, found := cachedContracts[cds] + cachedContractsMux.Unlock() + if !found { + address := hexutil.Encode(contractDesc) + data, err := b.ethCall(erc20NameSignature, address) + if err != nil { + return nil, err + } + name := parseErc20StringProperty(contractDesc, data) + if name != "" { + data, err = b.ethCall(erc20SymbolSignature, address) + if err != nil { + return nil, err + } + symbol := parseErc20StringProperty(contractDesc, data) + data, err = b.ethCall(erc20DecimalsSignature, address) + if err != nil { + return nil, err + } + if name == "" { + name = address + } + contract = &bchain.Erc20Contract{ + Contract: address, + Name: name, + Symbol: symbol, + } + d := parseErc20NumericProperty(contractDesc, data) + if d != nil { + contract.Decimals = int(uint8(d.Uint64())) + } else { + contract.Decimals = EtherAmountDecimalPoint + } + } else { + contract = nil + } + cachedContractsMux.Lock() + cachedContracts[cds] = contract + cachedContractsMux.Unlock() + } + return contract, nil +} + +// EthereumTypeGetErc20ContractBalance returns balance of ERC20 contract for given address +func (b *EthereumRPC) EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc bchain.AddressDescriptor) (*big.Int, error) { + addr := hexutil.Encode(addrDesc) + contract := hexutil.Encode(contractDesc) + req := erc20BalanceOf + "0000000000000000000000000000000000000000000000000000000000000000"[len(addr)-2:] + addr[2:] + data, err := b.ethCall(req, contract) + if err != nil { + return nil, err + } + r := parseErc20NumericProperty(contractDesc, data) + if r == nil { + return nil, errors.New("Invalid balance") + } + return r, nil +} diff --git a/bchain/coins/eth/erc20_test.go b/bchain/coins/eth/erc20_test.go new file mode 100644 index 00000000..60e19f9b --- /dev/null +++ b/bchain/coins/eth/erc20_test.go @@ -0,0 +1,183 @@ +// +build unittest + +package eth + +import ( + "blockbook/bchain" + "blockbook/tests/dbtestdata" + fmt "fmt" + "math/big" + "strings" + "testing" +) + +func TestErc20_erc20GetTransfersFromLog(t *testing.T) { + tests := []struct { + name string + args []*rpcLog + want []bchain.Erc20Transfer + wantErr bool + }{ + { + name: "1", + args: []*rpcLog{ + &rpcLog{ + Address: "0x76a45e8976499ab9ae223cc584019341d5a84e96", + Topics: []string{ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000002aacf811ac1a60081ea39f7783c0d26c500871a8", + "0x000000000000000000000000e9a5216ff992cfa01594d43501a56e12769eb9d2", + }, + Data: "0x0000000000000000000000000000000000000000000000000000000000000123", + }, + }, + want: []bchain.Erc20Transfer{ + { + Contract: "0x76a45e8976499ab9ae223cc584019341d5a84e96", + From: "0x2aacf811ac1a60081ea39f7783c0d26c500871a8", + To: "0xe9a5216ff992cfa01594d43501a56e12769eb9d2", + Tokens: *big.NewInt(0x123), + }, + }, + }, + { + name: "2", + args: []*rpcLog{ + &rpcLog{ // Transfer + Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", + Topics: []string{ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed", + "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d", + }, + Data: "0x0000000000000000000000000000000000000000000000006a8313d60b1f606b", + }, + &rpcLog{ // Transfer + Address: "0xc778417e063141139fce010982780140aa0cd5ab", + Topics: []string{ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d", + "0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed", + }, + Data: "0x000000000000000000000000000000000000000000000000000308fd0e798ac0", + }, + &rpcLog{ // not Transfer + Address: "0x479cc461fecd078f766ecc58533d6f69580cf3ac", + Topics: []string{ + "0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3", + "0x0000000000000000000000006f44cceb49b4a5812d54b6f494fc2febf25511ed", + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x5af266c0a89a07c1917deaa024414577e6c3c31c8907d079e13eb448c082594f", + }, + Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000", + }, + &rpcLog{ // not Transfer + Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", + Topics: []string{ + "0x0d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb3", + "0x0000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b", + "0xb0b69dad58df6032c3b266e19b1045b19c87acd2c06fb0c598090f44b8e263aa", + }, + Data: "0x0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d000000000000000000000000c778417e063141139fce010982780140aa0cd5ab0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + }, + }, + want: []bchain.Erc20Transfer{ + { + Contract: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", + From: "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed", + To: "0x4bda106325c335df99eab7fe363cac8a0ba2a24d", + Tokens: *big.NewInt(0x6a8313d60b1f606b), + }, + { + Contract: "0xc778417e063141139fce010982780140aa0cd5ab", + From: "0x4bda106325c335df99eab7fe363cac8a0ba2a24d", + To: "0x6f44cceb49b4a5812d54b6f494fc2febf25511ed", + Tokens: *big.NewInt(0x308fd0e798ac0), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := erc20GetTransfersFromLog(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("erc20GetTransfersFromLog error = %v, wantErr %v", err, tt.wantErr) + return + } + // the addresses could have different case + if strings.ToLower(fmt.Sprint(got)) != strings.ToLower(fmt.Sprint(tt.want)) { + t.Errorf("erc20GetTransfersFromLog = %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestErc20_parseErc20StringProperty(t *testing.T) { + tests := []struct { + name string + args string + want string + }{ + { + name: "1", + args: "0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000758504c4f44444500000000000000000000000000000000000000000000000000", + want: "XPLODDE", + }, + { + name: "2", + args: "0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000022426974436c617665202d20436f6e73756d657220416374697669747920546f6b656e00000000000000", + want: "BitClave - Consumer Activity Token", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseErc20StringProperty(nil, tt.args) + // the addresses could have different case + if got != tt.want { + t.Errorf("parseErc20StringProperty = %v, want %v", got, tt.want) + } + }) + } +} + +func TestErc20_erc20GetTransfersFromTx(t *testing.T) { + p := NewEthereumParser(1) + b := dbtestdata.GetTestEthereumTypeBlock1(p) + bn, _ := new(big.Int).SetString("21e19e0c9bab2400000", 16) + tests := []struct { + name string + args *rpcTransaction + want []bchain.Erc20Transfer + }{ + { + name: "0", + args: (b.Txs[0].CoinSpecificData.(completeTransaction)).Tx, + want: []bchain.Erc20Transfer{}, + }, + { + name: "1", + args: (b.Txs[1].CoinSpecificData.(completeTransaction)).Tx, + want: []bchain.Erc20Transfer{ + { + Contract: "0x4af4114f73d1c1c903ac9e0361b379d1291808a2", + From: "0x20cd153de35d469ba46127a0c8f18626b59a256a", + To: "0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f", + Tokens: *bn, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := erc20GetTransfersFromTx(tt.args) + if err != nil { + t.Errorf("erc20GetTransfersFromTx error = %v", err) + return + } + // the addresses could have different case + if strings.ToLower(fmt.Sprint(got)) != strings.ToLower(fmt.Sprint(tt.want)) { + t.Errorf("erc20GetTransfersFromTx = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index d76aa6bf..1118dc93 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -3,57 +3,95 @@ package eth import ( "blockbook/bchain" "encoding/hex" - "encoding/json" "math/big" "strconv" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/golang/protobuf/proto" "github.com/juju/errors" - - ethcommon "github.com/ethereum/go-ethereum/common" ) +// EthereumTypeAddressDescriptorLen - in case of EthereumType, the AddressDescriptor has fixed length +const EthereumTypeAddressDescriptorLen = 20 + +// EtherAmountDecimalPoint defines number of decimal points in Ether amounts +const EtherAmountDecimalPoint = 18 + // EthereumParser handle type EthereumParser struct { *bchain.BaseParser } // NewEthereumParser returns new EthereumParser instance -func NewEthereumParser() *EthereumParser { +func NewEthereumParser(b int) *EthereumParser { return &EthereumParser{&bchain.BaseParser{ - BlockAddressesToKeep: 0, - AmountDecimalPoint: 18, + BlockAddressesToKeep: b, + AmountDecimalPoint: EtherAmountDecimalPoint, }} } +type rpcHeader struct { + Hash string `json:"hash"` + ParentHash string `json:"parentHash"` + Difficulty string `json:"difficulty"` + Number string `json:"number"` + Time string `json:"timestamp"` + Size string `json:"size"` + Nonce string `json:"nonce"` +} + type rpcTransaction struct { - AccountNonce string `json:"nonce" gencodec:"required"` - Price string `json:"gasPrice" gencodec:"required"` - GasLimit string `json:"gas" gencodec:"required"` - To string `json:"to" rlp:"nil"` // nil means contract creation - Value string `json:"value" gencodec:"required"` - Payload string `json:"input" gencodec:"required"` - Hash ethcommon.Hash `json:"hash" rlp:"-"` - BlockNumber string `json:"blockNumber"` - BlockHash *ethcommon.Hash `json:"blockHash,omitempty"` - From string `json:"from"` - TransactionIndex string `json:"transactionIndex"` - // Signature values - V string `json:"v" gencodec:"required"` - R string `json:"r" gencodec:"required"` - S string `json:"s" gencodec:"required"` + AccountNonce string `json:"nonce"` + GasPrice string `json:"gasPrice"` + GasLimit string `json:"gas"` + To string `json:"to"` // nil means contract creation + Value string `json:"value"` + Payload string `json:"input"` + Hash string `json:"hash"` + BlockNumber string `json:"blockNumber"` + BlockHash string `json:"blockHash,omitempty"` + From string `json:"from"` + TransactionIndex string `json:"transactionIndex"` + // Signature values - ignored + // V string `json:"v"` + // R string `json:"r"` + // S string `json:"s"` } -type rpcBlock struct { - Hash ethcommon.Hash `json:"hash"` +type rpcLog struct { + Address string `json:"address"` + Topics []string `json:"topics"` + Data string `json:"data"` +} + +type rpcLogWithTxHash struct { + rpcLog + Hash string `json:"transactionHash"` +} + +type rpcReceipt struct { + GasUsed string `json:"gasUsed"` + Status string `json:"status"` + Logs []*rpcLog `json:"logs"` +} + +type rpcEtcReceipt struct { + GasUsed string `json:"gasUsed"` + Status int `json:"status"` + Logs []*rpcLog `json:"logs"` +} + +type completeTransaction struct { + Tx *rpcTransaction `json:"tx"` + Receipt *rpcReceipt `json:"receipt,omitempty"` +} + +type rpcBlockTransactions struct { Transactions []rpcTransaction `json:"transactions"` - UncleHashes []ethcommon.Hash `json:"uncles"` } -func ethHashToHash(h ethcommon.Hash) string { - return h.Hex() +type rpcBlockTxids struct { + Transactions []string `json:"transactions"` } func ethNumber(n string) (int64, error) { @@ -63,8 +101,8 @@ func ethNumber(n string) (int64, error) { return 0, errors.Errorf("Not a number: '%v'", n) } -func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirmations uint32) (*bchain.Tx, error) { - txid := ethHashToHash(tx.Hash) +func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, blocktime int64, confirmations uint32) (*bchain.Tx, error) { + txid := tx.Hash var ( fa, ta []string err error @@ -75,15 +113,10 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma if len(tx.To) > 2 { ta = []string{tx.To} } - // temporarily, the complete rpcTransaction without BlockHash is marshalled and hex encoded to bchain.Tx.Hex - bh := tx.BlockHash - tx.BlockHash = nil - b, err := json.Marshal(tx) - if err != nil { - return nil, err + ct := completeTransaction{ + Tx: tx, + Receipt: receipt, } - tx.BlockHash = bh - h := hex.EncodeToString(b) vs, err := hexutil.DecodeBig(tx.Value) if err != nil { return nil, err @@ -91,7 +124,7 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma return &bchain.Tx{ Blocktime: blocktime, Confirmations: confirmations, - Hex: h, + // Hex // LockTime Time: blocktime, Txid: txid, @@ -115,6 +148,7 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, blocktime int64, confirma }, }, }, + CoinSpecificData: ct, }, nil } @@ -136,12 +170,9 @@ func (p *EthereumParser) GetAddrDescFromAddress(address string) (bchain.AddressD if has0xPrefix(address) { address = address[2:] } - if len(address) == 0 { + if len(address) != EthereumTypeAddressDescriptorLen*2 { return nil, bchain.ErrAddressMissing } - if len(address)&1 == 1 { - address = "0" + address - } return hex.DecodeString(address) } @@ -179,83 +210,137 @@ func hexEncodeBig(b []byte) string { // PackTx packs transaction to byte array func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { - b, err := hex.DecodeString(tx.Hex) - if err != nil { - return nil, err - } - var r rpcTransaction + var err error var n uint64 - err = json.Unmarshal(b, &r) - if err != nil { - return nil, err + r, ok := tx.CoinSpecificData.(completeTransaction) + if !ok { + return nil, errors.New("Missing CoinSpecificData") } - pt := &ProtoTransaction{} - if pt.AccountNonce, err = hexutil.DecodeUint64(r.AccountNonce); err != nil { - return nil, errors.Annotatef(err, "AccountNonce %v", r.AccountNonce) + pt := &ProtoCompleteTransaction{} + pt.Tx = &ProtoCompleteTransaction_TxType{} + if pt.Tx.AccountNonce, err = hexutil.DecodeUint64(r.Tx.AccountNonce); err != nil { + return nil, errors.Annotatef(err, "AccountNonce %v", r.Tx.AccountNonce) } - if n, err = hexutil.DecodeUint64(r.BlockNumber); err != nil { - return nil, errors.Annotatef(err, "BlockNumber %v", r.BlockNumber) + // pt.BlockNumber = height + if n, err = hexutil.DecodeUint64(r.Tx.BlockNumber); err != nil { + return nil, errors.Annotatef(err, "BlockNumber %v", r.Tx.BlockNumber) } pt.BlockNumber = uint32(n) pt.BlockTime = uint64(blockTime) - if pt.From, err = hexDecode(r.From); err != nil { - return nil, errors.Annotatef(err, "From %v", r.From) + if pt.Tx.From, err = hexDecode(r.Tx.From); err != nil { + return nil, errors.Annotatef(err, "From %v", r.Tx.From) } - if pt.GasLimit, err = hexutil.DecodeUint64(r.GasLimit); err != nil { - return nil, errors.Annotatef(err, "GasLimit %v", r.GasLimit) + if pt.Tx.GasLimit, err = hexutil.DecodeUint64(r.Tx.GasLimit); err != nil { + return nil, errors.Annotatef(err, "GasLimit %v", r.Tx.GasLimit) } - pt.Hash = r.Hash.Bytes() - if pt.Payload, err = hexDecode(r.Payload); err != nil { - return nil, errors.Annotatef(err, "Payload %v", r.Payload) + if pt.Tx.Hash, err = hexDecode(r.Tx.Hash); err != nil { + return nil, errors.Annotatef(err, "Hash %v", r.Tx.Hash) } - if pt.Price, err = hexDecodeBig(r.Price); err != nil { - return nil, errors.Annotatef(err, "Price %v", r.Price) + if pt.Tx.Payload, err = hexDecode(r.Tx.Payload); err != nil { + return nil, errors.Annotatef(err, "Payload %v", r.Tx.Payload) } - if pt.R, err = hexDecodeBig(r.R); err != nil { - return nil, errors.Annotatef(err, "R %v", r.R) + if pt.Tx.GasPrice, err = hexDecodeBig(r.Tx.GasPrice); err != nil { + return nil, errors.Annotatef(err, "Price %v", r.Tx.GasPrice) } - if pt.S, err = hexDecodeBig(r.S); err != nil { - return nil, errors.Annotatef(err, "S %v", r.S) + // if pt.R, err = hexDecodeBig(r.R); err != nil { + // return nil, errors.Annotatef(err, "R %v", r.R) + // } + // if pt.S, err = hexDecodeBig(r.S); err != nil { + // return nil, errors.Annotatef(err, "S %v", r.S) + // } + // if pt.V, err = hexDecodeBig(r.V); err != nil { + // return nil, errors.Annotatef(err, "V %v", r.V) + // } + if pt.Tx.To, err = hexDecode(r.Tx.To); err != nil { + return nil, errors.Annotatef(err, "To %v", r.Tx.To) } - if pt.V, err = hexDecodeBig(r.V); err != nil { - return nil, errors.Annotatef(err, "V %v", r.V) + if n, err = hexutil.DecodeUint64(r.Tx.TransactionIndex); err != nil { + return nil, errors.Annotatef(err, "TransactionIndex %v", r.Tx.TransactionIndex) } - if pt.To, err = hexDecode(r.To); err != nil { - return nil, errors.Annotatef(err, "To %v", r.To) + pt.Tx.TransactionIndex = uint32(n) + if pt.Tx.Value, err = hexDecodeBig(r.Tx.Value); err != nil { + return nil, errors.Annotatef(err, "Value %v", r.Tx.Value) } - if n, err = hexutil.DecodeUint64(r.TransactionIndex); err != nil { - return nil, errors.Annotatef(err, "TransactionIndex %v", r.TransactionIndex) - } - pt.TransactionIndex = uint32(n) - if pt.Value, err = hexDecodeBig(r.Value); err != nil { - return nil, errors.Annotatef(err, "Value %v", r.Value) + if r.Receipt != nil { + pt.Receipt = &ProtoCompleteTransaction_ReceiptType{} + if pt.Receipt.GasUsed, err = hexDecodeBig(r.Receipt.GasUsed); err != nil { + return nil, errors.Annotatef(err, "GasUsed %v", r.Receipt.GasUsed) + } + if pt.Receipt.Status, err = hexDecodeBig(r.Receipt.Status); err != nil { + return nil, errors.Annotatef(err, "Status %v", r.Receipt.Status) + } + ptLogs := make([]*ProtoCompleteTransaction_ReceiptType_LogType, len(r.Receipt.Logs)) + for i, l := range r.Receipt.Logs { + a, err := hexutil.Decode(l.Address) + if err != nil { + return nil, errors.Annotatef(err, "Address cannot be decoded %v", l) + } + d, err := hexutil.Decode(l.Data) + if err != nil { + return nil, errors.Annotatef(err, "Data cannot be decoded %v", l) + } + t := make([][]byte, len(l.Topics)) + for j, s := range l.Topics { + t[j], err = hexutil.Decode(s) + if err != nil { + return nil, errors.Annotatef(err, "Topic cannot be decoded %v", l) + } + } + ptLogs[i] = &ProtoCompleteTransaction_ReceiptType_LogType{ + Address: a, + Data: d, + Topics: t, + } + + } + pt.Receipt.Log = ptLogs } return proto.Marshal(pt) } // UnpackTx unpacks transaction from byte array func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { - var pt ProtoTransaction + var pt ProtoCompleteTransaction err := proto.Unmarshal(buf, &pt) if err != nil { return nil, 0, err } - r := rpcTransaction{ - AccountNonce: hexutil.EncodeUint64(pt.AccountNonce), - BlockNumber: hexutil.EncodeUint64(uint64(pt.BlockNumber)), - From: hexutil.Encode(pt.From), - GasLimit: hexutil.EncodeUint64(pt.GasLimit), - Hash: ethcommon.BytesToHash(pt.Hash), - Payload: hexutil.Encode(pt.Payload), - Price: hexEncodeBig(pt.Price), - R: hexEncodeBig(pt.R), - S: hexEncodeBig(pt.S), - V: hexEncodeBig(pt.V), - To: hexutil.Encode(pt.To), - TransactionIndex: hexutil.EncodeUint64(uint64(pt.TransactionIndex)), - Value: hexEncodeBig(pt.Value), + rt := rpcTransaction{ + AccountNonce: hexutil.EncodeUint64(pt.Tx.AccountNonce), + BlockNumber: hexutil.EncodeUint64(uint64(pt.BlockNumber)), + From: hexutil.Encode(pt.Tx.From), + GasLimit: hexutil.EncodeUint64(pt.Tx.GasLimit), + Hash: hexutil.Encode(pt.Tx.Hash), + Payload: hexutil.Encode(pt.Tx.Payload), + GasPrice: hexEncodeBig(pt.Tx.GasPrice), + // R: hexEncodeBig(pt.R), + // S: hexEncodeBig(pt.S), + // V: hexEncodeBig(pt.V), + To: hexutil.Encode(pt.Tx.To), + TransactionIndex: hexutil.EncodeUint64(uint64(pt.Tx.TransactionIndex)), + Value: hexEncodeBig(pt.Tx.Value), } - tx, err := p.ethTxToTx(&r, int64(pt.BlockTime), 0) + var rr *rpcReceipt + if pt.Receipt != nil { + logs := make([]*rpcLog, len(pt.Receipt.Log)) + for i, l := range pt.Receipt.Log { + topics := make([]string, len(l.Topics)) + for j, t := range l.Topics { + topics[j] = hexutil.Encode(t) + } + logs[i] = &rpcLog{ + Address: hexutil.Encode(l.Address), + Data: hexutil.Encode(l.Data), + Topics: topics, + } + } + rr = &rpcReceipt{ + GasUsed: hexEncodeBig(pt.Receipt.GasUsed), + Status: hexEncodeBig(pt.Receipt.Status), + Logs: logs, + } + } + tx, err := p.ethTxToTx(&rt, rr, int64(pt.BlockTime), 0) if err != nil { return nil, 0, err } @@ -293,7 +378,81 @@ func (p *EthereumParser) UnpackBlockHash(buf []byte) (string, error) { return hexutil.Encode(buf), nil } -// IsUTXOChain returns true if the block chain is UTXO type, otherwise false -func (p *EthereumParser) IsUTXOChain() bool { - return false +// GetChainType returns EthereumType +func (p *EthereumParser) GetChainType() bchain.ChainType { + return bchain.ChainEthereumType +} + +// GetHeightFromTx returns ethereum specific data from bchain.Tx +func GetHeightFromTx(tx *bchain.Tx) (uint32, error) { + var bn string + csd, ok := tx.CoinSpecificData.(completeTransaction) + if !ok { + return 0, errors.New("Missing CoinSpecificData") + } + bn = csd.Tx.BlockNumber + n, err := hexutil.DecodeUint64(bn) + if err != nil { + return 0, errors.Annotatef(err, "BlockNumber %v", bn) + } + return uint32(n), nil +} + +// EthereumTypeGetErc20FromTx returns Erc20 data from bchain.Tx +func (p *EthereumParser) EthereumTypeGetErc20FromTx(tx *bchain.Tx) ([]bchain.Erc20Transfer, error) { + var r []bchain.Erc20Transfer + var err error + csd, ok := tx.CoinSpecificData.(completeTransaction) + if ok { + if csd.Receipt != nil { + r, err = erc20GetTransfersFromLog(csd.Receipt.Logs) + } else { + r, err = erc20GetTransfersFromTx(csd.Tx) + } + if err != nil { + return nil, err + } + } + return r, nil +} + +const ( + txStatusUnknown = iota - 2 + txStatusPending + txStatusFailure + txStatusOK +) + +// EthereumTxData contains ethereum specific transaction data +type EthereumTxData struct { + Status int `json:"status"` // 1 OK, 0 Fail, -1 pending, -2 unknown + Nonce uint64 `json:"nonce"` + GasLimit *big.Int `json:"gaslimit"` + GasUsed *big.Int `json:"gasused"` + GasPrice *big.Int `json:"gasprice"` +} + +// GetEthereumTxData returns EthereumTxData from bchain.Tx +func GetEthereumTxData(tx *bchain.Tx) *EthereumTxData { + etd := EthereumTxData{Status: txStatusPending} + csd, ok := tx.CoinSpecificData.(completeTransaction) + if ok { + if csd.Tx != nil { + etd.Nonce, _ = hexutil.DecodeUint64(csd.Tx.AccountNonce) + etd.GasLimit, _ = hexutil.DecodeBig(csd.Tx.GasLimit) + etd.GasPrice, _ = hexutil.DecodeBig(csd.Tx.GasPrice) + } + if csd.Receipt != nil { + switch csd.Receipt.Status { + case "0x1": + etd.Status = txStatusOK + case "": // old transactions did not set status + etd.Status = txStatusUnknown + default: + etd.Status = txStatusFailure + } + etd.GasUsed, _ = hexutil.DecodeBig(csd.Receipt.GasUsed) + } + } + return &etd } diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index 2a7ee8bf..7012d061 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -4,7 +4,9 @@ package eth import ( "blockbook/bchain" + "blockbook/tests/dbtestdata" "encoding/hex" + "fmt" "math/big" "reflect" "testing" @@ -31,9 +33,10 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) { want: "47526228d673e9f079630d6cdaff5a2ed13e0e60", }, { - name: "odd address", - args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"}, - want: "07526228d673e9f079630d6cdaff5a2ed13e0e60", + name: "address of wrong length", + args: args{address: "7526228d673e9f079630d6cdaff5a2ed13e0e60"}, + want: "", + wantErr: true, }, { name: "ErrAddressMissing", @@ -50,7 +53,7 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewEthereumParser() + p := NewEthereumParser(1) got, err := p.GetAddrDescFromAddress(tt.args.address) if (err != nil) != tt.wantErr { t.Errorf("EthParser.GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) @@ -64,38 +67,13 @@ func TestEthParser_GetAddrDescFromAddress(t *testing.T) { } } -var ( - testTx1, testTx2 bchain.Tx - testTxPacked1 = "08aebf0a1205012a05f20018a0f73622081234567890abcdef2a24f025caaf00000000000000000000000000000000000000000000000000000000000002253220e6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d38f095af014092f4c1d5054a14682b7903a11098cf770c7aef4aa02a85b3f3601a5214dacc9c61754a0c4616fc5323dc946e89eb272302580162011b6a201bd40a31122c03918df6d166d740a6a3a22f08a25934ceb1688c62977661c80c7220607fbc15c1f7995a4258f5a9bccc63b040362d1991d5efe1361c56222e4ca89f" - testTxPacked2 = "08ece40212050430e234001888a4012201213220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b38889eaf0140fa83c3d5054a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f52143e3a3d69dc66ba10737f531ed088954a9ec89d97580a6201296a20f7161c170d43573ad9c8d701cdaf714ff2a548a562b0dc639230d17889fcd40572203c4977fc90385a27efa0032e17b49fd575b2826cb56e3d1ecf21524f2a94f915" -) +var testTx1, testTx2 bchain.Tx func init() { testTx1 = bchain.Tx{ - Blocktime: 1521515026, - Hex: "7b226e6f6e6365223a2230783239666165222c226761735072696365223a223078313261303566323030222c22676173223a2230786462626130222c22746f223a22307836383262373930336131313039386366373730633761656634616130326138356233663336303161222c2276616c7565223a22307831323334353637383930616263646566222c22696e707574223a223078663032356361616630303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030323235222c2268617368223a22307865366231363864366262336438656437386530336462663832386236626664316662363133663665313239636261363234393634393834353533373234633564222c22626c6f636b4e756d626572223a223078326263616630222c2266726f6d223a22307864616363396336313735346130633436313666633533323364633934366538396562323732333032222c227472616e73616374696f6e496e646578223a22307831222c2276223a2230783162222c2272223a22307831626434306133313132326330333931386466366431363664373430613661336132326630386132353933346365623136383863363239373736363163383063222c2273223a22307836303766626331356331663739393561343235386635613962636363363362303430333632643139393164356566653133363163353632323265346361383966227d", - Time: 1521515026, - Txid: "0xe6b168d6bb3d8ed78e03dbf828b6bfd1fb613f6e129cba624964984553724c5d", - Vin: []bchain.Vin{ - { - Addresses: []string{"0xdacc9c61754a0c4616fc5323dc946e89eb272302"}, - }, - }, - Vout: []bchain.Vout{ - { - ValueSat: *big.NewInt(1311768467294899695), - ScriptPubKey: bchain.ScriptPubKey{ - Addresses: []string{"0x682b7903a11098cf770c7aef4aa02a85b3f3601a"}, - }, - }, - }, - } - - testTx2 = bchain.Tx{ - Blocktime: 1521533434, - Hex: "7b226e6f6e6365223a22307862323663222c226761735072696365223a223078343330653233343030222c22676173223a22307835323038222c22746f223a22307835353565653131666264646330653439613962616233353861383934316164393566666462343866222c2276616c7565223a2230783231222c22696e707574223a223078222c2268617368223a22307863643634373135313535326235313332623261656637633962653030646336663733616663353930316464653135376161623133313333356261616138353362222c22626c6f636b4e756d626572223a223078326263663038222c2266726f6d223a22307833653361336436396463363662613130373337663533316564303838393534613965633839643937222c227472616e73616374696f6e496e646578223a22307861222c2276223a2230783239222c2272223a22307866373136316331373064343335373361643963386437303163646166373134666632613534386135363262306463363339323330643137383839666364343035222c2273223a22307833633439373766633930333835613237656661303033326531376234396664353735623238323663623536653364316563663231353234663261393466393135227d", - Time: 1521533434, + Blocktime: 1534858022, + Time: 1534858022, Txid: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", Vin: []bchain.Vin{ { @@ -104,12 +82,78 @@ func init() { }, Vout: []bchain.Vout{ { - ValueSat: *big.NewInt(33), + ValueSat: *big.NewInt(1999622000000000000), ScriptPubKey: bchain.ScriptPubKey{ Addresses: []string{"0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f"}, }, }, }, + CoinSpecificData: completeTransaction{ + Tx: &rpcTransaction{ + AccountNonce: "0xb26c", + GasPrice: "0x430e23400", + GasLimit: "0x5208", + To: "0x555ee11fbddc0e49a9bab358a8941ad95ffdb48f", + Value: "0x1bc0159d530e6000", + Payload: "0x", + Hash: "0xcd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b", + BlockNumber: "0x41eee8", + From: "0x3e3a3d69dc66ba10737f531ed088954a9ec89d97", + TransactionIndex: "0xa", + }, + Receipt: &rpcReceipt{ + GasUsed: "0x5208", + Status: "0x1", + Logs: []*rpcLog{}, + }, + }, + } + + testTx2 = bchain.Tx{ + Blocktime: 1534858022, + Time: 1534858022, + Txid: "0xa9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b101", + Vin: []bchain.Vin{ + { + Addresses: []string{"0x20cd153de35d469ba46127a0c8f18626b59a256a"}, + }, + }, + Vout: []bchain.Vout{ + { + ValueSat: *big.NewInt(0), + ScriptPubKey: bchain.ScriptPubKey{ + Addresses: []string{"0x4af4114f73d1c1c903ac9e0361b379d1291808a2"}, + }, + }, + }, + CoinSpecificData: completeTransaction{ + Tx: &rpcTransaction{ + AccountNonce: "0xd0", + GasPrice: "0x9502f9000", + GasLimit: "0x130d5", + To: "0x4af4114f73d1c1c903ac9e0361b379d1291808a2", + Value: "0x0", + Payload: "0xa9059cbb000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f00000000000000000000000000000000000000000000021e19e0c9bab2400000", + Hash: "0xa9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b101", + BlockNumber: "0x41eee8", + From: "0x20cd153de35d469ba46127a0c8f18626b59a256a", + TransactionIndex: "0x0"}, + Receipt: &rpcReceipt{ + GasUsed: "0xcb39", + Status: "0x1", + Logs: []*rpcLog{ + &rpcLog{ + Address: "0x4af4114f73d1c1c903ac9e0361b379d1291808a2", + Data: "0x00000000000000000000000000000000000000000000021e19e0c9bab2400000", + Topics: []string{ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x00000000000000000000000020cd153de35d469ba46127a0c8f18626b59a256a", + "0x000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f", + }, + }, + }, + }, + }, } } @@ -130,22 +174,22 @@ func TestEthereumParser_PackTx(t *testing.T) { name: "1", args: args{ tx: &testTx1, - height: 2870000, - blockTime: 1521515026, + height: 4321000, + blockTime: 1534858022, }, - want: testTxPacked1, + want: dbtestdata.EthTx1Packed, }, { name: "2", args: args{ tx: &testTx2, - height: 2871048, - blockTime: 1521533434, + height: 4321000, + blockTime: 1534858022, }, - want: testTxPacked2, + want: dbtestdata.EthTx2Packed, }, } - p := NewEthereumParser() + p := NewEthereumParser(1) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := p.PackTx(tt.args.tx, tt.args.height, tt.args.blockTime) @@ -175,18 +219,18 @@ func TestEthereumParser_UnpackTx(t *testing.T) { }{ { name: "1", - args: args{hex: testTxPacked1}, + args: args{hex: dbtestdata.EthTx1Packed}, want: &testTx1, - want1: 2870000, + want1: 4321000, }, { name: "2", - args: args{hex: testTxPacked2}, + args: args{hex: dbtestdata.EthTx2Packed}, want: &testTx2, - want1: 2871048, + want1: 4321000, }, } - p := NewEthereumParser() + p := NewEthereumParser(1) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b, err := hex.DecodeString(tt.args.hex) @@ -198,8 +242,22 @@ func TestEthereumParser_UnpackTx(t *testing.T) { t.Errorf("EthereumParser.UnpackTx() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("EthereumParser.UnpackTx() got = %v, want %v", got, tt.want) + // DeepEqual has problems with pointers in completeTransaction + gs := got.CoinSpecificData.(completeTransaction) + ws := tt.want.CoinSpecificData.(completeTransaction) + gc := *got + wc := *tt.want + gc.CoinSpecificData = nil + wc.CoinSpecificData = nil + if fmt.Sprint(gc) != fmt.Sprint(wc) { + // if !reflect.DeepEqual(gc, wc) { + t.Errorf("EthereumParser.UnpackTx() gc got = %+v, want %+v", gc, wc) + } + if !reflect.DeepEqual(gs.Tx, ws.Tx) { + t.Errorf("EthereumParser.UnpackTx() gs.Tx got = %+v, want %+v", gs.Tx, ws.Tx) + } + if !reflect.DeepEqual(gs.Receipt, ws.Receipt) { + t.Errorf("EthereumParser.UnpackTx() gs.Receipt got = %+v, want %+v", gs.Receipt, ws.Receipt) } if got1 != tt.want1 { t.Errorf("EthereumParser.UnpackTx() got1 = %v, want %v", got1, tt.want1) diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 8a5c98c9..8aba3598 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -10,14 +10,13 @@ import ( "sync" "time" - "github.com/golang/glog" - "github.com/juju/errors" - ethereum "github.com/ethereum/go-ethereum" ethcommon "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" + "github.com/golang/glog" + "github.com/juju/errors" ) // EthereumNet type specifies the type of ethereum network @@ -30,31 +29,34 @@ const ( TestNet EthereumNet = 3 ) +// Configuration represents json config file type Configuration struct { - CoinName string `json:"coin_name"` - CoinShortcut string `json:"coin_shortcut"` - RPCURL string `json:"rpc_url"` - RPCTimeout int `json:"rpc_timeout"` + CoinName string `json:"coin_name"` + CoinShortcut string `json:"coin_shortcut"` + RPCURL string `json:"rpc_url"` + RPCTimeout int `json:"rpc_timeout"` + BlockAddressesToKeep int `json:"block_addresses_to_keep"` } // EthereumRPC is an interface to JSON-RPC eth service. type EthereumRPC struct { - client *ethclient.Client - rpc *rpc.Client - timeout time.Duration - Parser *EthereumParser - Testnet bool - Network string - Mempool *bchain.NonUTXOMempool - bestHeaderMu sync.Mutex - bestHeader *ethtypes.Header - bestHeaderTime time.Time - chanNewBlock chan *ethtypes.Header - newBlockSubscription *rpc.ClientSubscription - chanNewTx chan ethcommon.Hash - newTxSubscription *rpc.ClientSubscription - ChainConfig *Configuration - isETC bool + *bchain.BaseChain + client *ethclient.Client + rpc *rpc.Client + timeout time.Duration + Parser *EthereumParser + Mempool *bchain.MempoolEthereumType + bestHeaderLock sync.Mutex + bestHeader *ethtypes.Header + bestHeaderTime time.Time + chanNewBlock chan *ethtypes.Header + newBlockSubscription *rpc.ClientSubscription + chanNewTx chan ethcommon.Hash + newTxSubscription *rpc.ClientSubscription + pendingTransactions map[string]struct{} + pendingTransactionsLock sync.Mutex + ChainConfig *Configuration + isETC bool } // NewEthereumRPC returns new EthRPC instance. @@ -65,6 +67,10 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification if err != nil { return nil, errors.Annotatef(err, "Invalid configuration file") } + // keep at least 100 mappings block->addresses to allow rollback + if c.BlockAddressesToKeep < 100 { + c.BlockAddressesToKeep = 100 + } rc, err := rpc.Dial(c.RPCURL) if err != nil { return nil, err @@ -72,13 +78,15 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification ec := ethclient.NewClient(rc) s := &EthereumRPC{ - client: ec, - rpc: rc, - ChainConfig: &c, + BaseChain: &bchain.BaseChain{}, + client: ec, + rpc: rc, + ChainConfig: &c, + pendingTransactions: make(map[string]struct{}), } // always create parser - s.Parser = NewEthereumParser() + s.Parser = NewEthereumParser(c.BlockAddressesToKeep) s.timeout = time.Duration(c.RPCTimeout) * time.Second // detect ethereum classic @@ -95,10 +103,10 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification } glog.V(2).Info("rpc: new block header ", h.Number) // update best header to the new header - s.bestHeaderMu.Lock() + s.bestHeaderLock.Lock() s.bestHeader = h s.bestHeaderTime = time.Now() - s.bestHeaderMu.Unlock() + s.bestHeaderLock.Unlock() // notify blockbook pushHandler(bchain.NotificationNewBlock) } @@ -113,9 +121,13 @@ func NewEthereumRPC(config json.RawMessage, pushHandler func(bchain.Notification if !ok { break } + hex := t.Hex() if glog.V(2) { - glog.Info("rpc: new tx ", t.Hex()) + glog.Info("rpc: new tx ", hex) } + s.pendingTransactionsLock.Lock() + s.pendingTransactions[hex] = struct{}{} + s.pendingTransactionsLock.Unlock() pushHandler(bchain.NotificationNewTx) } }() @@ -185,7 +197,7 @@ func (b *EthereumRPC) Initialize() error { } // create mempool - b.Mempool = bchain.NewNonUTXOMempool(b) + b.Mempool = bchain.NewMempoolEthereumType(b) return nil } @@ -245,18 +257,12 @@ func (b *EthereumRPC) Shutdown(ctx context.Context) error { return nil } -func (b *EthereumRPC) IsTestnet() bool { - return b.Testnet -} - -func (b *EthereumRPC) GetNetworkName() string { - return b.Network -} - +// GetCoinName returns coin name func (b *EthereumRPC) GetCoinName() string { return b.ChainConfig.CoinName } +// GetSubversion returns empty string, ethereum does not have subversion func (b *EthereumRPC) GetSubversion() string { return "" } @@ -269,20 +275,36 @@ func (b *EthereumRPC) GetChainInfo() (*bchain.ChainInfo, error) { if err != nil { return nil, err } - rv := &bchain.ChainInfo{} + h, err := b.getBestHeader() + if err != nil { + return nil, err + } + var ver, protocol string + if err := b.rpc.CallContext(ctx, &ver, "web3_clientVersion"); err != nil { + return nil, err + } + if err := b.rpc.CallContext(ctx, &protocol, "eth_protocolVersion"); err != nil { + return nil, err + } + rv := &bchain.ChainInfo{ + Blocks: int(h.Number.Int64()), + Bestblockhash: h.Hash().Hex(), + Difficulty: h.Difficulty.String(), + Version: ver, + ProtocolVersion: protocol, + } idi := int(id.Uint64()) if idi == 1 { rv.Chain = "mainnet" } else { rv.Chain = "testnet " + strconv.Itoa(idi) } - // TODO - return more information about the chain return rv, nil } func (b *EthereumRPC) getBestHeader() (*ethtypes.Header, error) { - b.bestHeaderMu.Lock() - defer b.bestHeaderMu.Unlock() + b.bestHeaderLock.Lock() + defer b.bestHeaderLock.Unlock() // ETC does not have newBlocks subscription, bestHeader must be updated very often (each 1 second) if b.isETC { if b.bestHeaderTime.Add(1 * time.Second).Before(time.Now()) { @@ -302,23 +324,25 @@ func (b *EthereumRPC) getBestHeader() (*ethtypes.Header, error) { return b.bestHeader, nil } +// GetBestBlockHash returns hash of the tip of the best-block-chain func (b *EthereumRPC) GetBestBlockHash() (string, error) { h, err := b.getBestHeader() if err != nil { return "", err } - return ethHashToHash(h.Hash()), nil + return h.Hash().Hex(), nil } +// GetBestBlockHeight returns height of the tip of the best-block-chain func (b *EthereumRPC) GetBestBlockHeight() (uint32, error) { h, err := b.getBestHeader() if err != nil { return 0, err } - // TODO - can it grow over 2^32 ? return uint32(h.Number.Uint64()), nil } +// GetBlockHash returns hash of block in best-block-chain at given height func (b *EthereumRPC) GetBlockHash(height uint32) (string, error) { var n big.Int n.SetUint64(uint64(height)) @@ -331,36 +355,47 @@ func (b *EthereumRPC) GetBlockHash(height uint32) (string, error) { } return "", errors.Annotatef(err, "height %v", height) } - return ethHashToHash(h.Hash()), nil + return h.Hash().Hex(), nil } -func (b *EthereumRPC) ethHeaderToBlockHeader(h *ethtypes.Header) (*bchain.BlockHeader, error) { - hn := h.Number.Uint64() - c, err := b.computeConfirmations(hn) +func (b *EthereumRPC) ethHeaderToBlockHeader(h *rpcHeader) (*bchain.BlockHeader, error) { + height, err := ethNumber(h.Number) + if err != nil { + return nil, err + } + c, err := b.computeConfirmations(uint64(height)) + if err != nil { + return nil, err + } + time, err := ethNumber(h.Time) + if err != nil { + return nil, err + } + size, err := ethNumber(h.Size) if err != nil { return nil, err } return &bchain.BlockHeader{ - Hash: ethHashToHash(h.Hash()), - Height: uint32(hn), + Hash: h.Hash, + Prev: h.ParentHash, + Height: uint32(height), Confirmations: int(c), - Time: int64(h.Time.Uint64()), - // Next - // Prev + Time: time, + Size: int(size), }, nil } +// GetBlockHeader returns header of block with given hash func (b *EthereumRPC) GetBlockHeader(hash string) (*bchain.BlockHeader, error) { - ctx, cancel := context.WithTimeout(context.Background(), b.timeout) - defer cancel() - h, err := b.client.HeaderByHash(ctx, ethcommon.HexToHash(hash)) + raw, err := b.getBlockRaw(hash, 0, false) if err != nil { - if err == ethereum.NotFound { - return nil, bchain.ErrBlockNotFound - } + return nil, err + } + var h rpcHeader + if err := json.Unmarshal(raw, &h); err != nil { return nil, errors.Annotatef(err, "hash %v", hash) } - return b.ethHeaderToBlockHeader(h) + return b.ethHeaderToBlockHeader(&h) } func (b *EthereumRPC) computeConfirmations(n uint64) (uint32, error) { @@ -373,61 +408,82 @@ func (b *EthereumRPC) computeConfirmations(n uint64) (uint32, error) { return uint32(bn - n + 1), nil } -// GetBlock returns block with given hash or height, hash has precedence if both passed -func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { +func (b *EthereumRPC) getBlockRaw(hash string, height uint32, fullTxs bool) (json.RawMessage, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() var raw json.RawMessage var err error if hash != "" { - err = b.rpc.CallContext(ctx, &raw, "eth_getBlockByHash", ethcommon.HexToHash(hash), true) + if hash == "pending" { + err = b.rpc.CallContext(ctx, &raw, "eth_getBlockByNumber", hash, fullTxs) + } else { + err = b.rpc.CallContext(ctx, &raw, "eth_getBlockByHash", ethcommon.HexToHash(hash), fullTxs) + } } else { - - err = b.rpc.CallContext(ctx, &raw, "eth_getBlockByNumber", fmt.Sprintf("%#x", height), true) + err = b.rpc.CallContext(ctx, &raw, "eth_getBlockByNumber", fmt.Sprintf("%#x", height), fullTxs) } if err != nil { return nil, errors.Annotatef(err, "hash %v, height %v", hash, height) } else if len(raw) == 0 { return nil, bchain.ErrBlockNotFound } - // Decode header and transactions. - var head *ethtypes.Header - var body rpcBlock + return raw, nil +} + +func (b *EthereumRPC) getERC20EventsForBlock(blockNumber string) (map[string][]*rpcLog, error) { + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + var logs []rpcLogWithTxHash + err := b.rpc.CallContext(ctx, &logs, "eth_getLogs", map[string]interface{}{ + "fromBlock": blockNumber, + "toBlock": blockNumber, + "topics": []string{erc20TransferEventSignature}, + }) + if err != nil { + return nil, errors.Annotatef(err, "blockNumber %v", blockNumber) + } + r := make(map[string][]*rpcLog) + for i := range logs { + l := &logs[i] + r[l.Hash] = append(r[l.Hash], &l.rpcLog) + } + return r, nil +} + +// GetBlock returns block with given hash or height, hash has precedence if both passed +func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { + raw, err := b.getBlockRaw(hash, height, true) + if err != nil { + return nil, err + } + var head rpcHeader if err := json.Unmarshal(raw, &head); err != nil { return nil, errors.Annotatef(err, "hash %v, height %v", hash, height) } - if head == nil { - return nil, bchain.ErrBlockNotFound - } + var body rpcBlockTransactions if err := json.Unmarshal(raw, &body); err != nil { return nil, errors.Annotatef(err, "hash %v, height %v", hash, height) } - // Quick-verify transaction and uncle lists. This mostly helps with debugging the server. - if head.UncleHash == ethtypes.EmptyUncleHash && len(body.UncleHashes) > 0 { - return nil, errors.Annotatef(fmt.Errorf("server returned non-empty uncle list but block header indicates no uncles"), "hash %v, height %v", hash, height) - } - if head.UncleHash != ethtypes.EmptyUncleHash && len(body.UncleHashes) == 0 { - return nil, errors.Annotatef(fmt.Errorf("server returned empty uncle list but block header indicates uncles"), "hash %v, height %v", hash, height) - } - if head.TxHash == ethtypes.EmptyRootHash && len(body.Transactions) > 0 { - return nil, errors.Annotatef(fmt.Errorf("server returned non-empty transaction list but block header indicates no transactions"), "hash %v, height %v", hash, height) - } - if head.TxHash != ethtypes.EmptyRootHash && len(body.Transactions) == 0 { - return nil, errors.Annotatef(fmt.Errorf("server returned empty transaction list but block header indicates transactions"), "hash %v, height %v", hash, height) - } - bbh, err := b.ethHeaderToBlockHeader(head) + bbh, err := b.ethHeaderToBlockHeader(&head) if err != nil { return nil, errors.Annotatef(err, "hash %v, height %v", hash, height) } - // TODO - this is probably not the correct size - bbh.Size = len(raw) + // get ERC20 events + logs, err := b.getERC20EventsForBlock(head.Number) + if err != nil { + return nil, err + } btxs := make([]bchain.Tx, len(body.Transactions)) - for i, tx := range body.Transactions { - btx, err := b.Parser.ethTxToTx(&tx, int64(head.Time.Uint64()), uint32(bbh.Confirmations)) + for i := range body.Transactions { + tx := &body.Transactions[i] + btx, err := b.Parser.ethTxToTx(tx, &rpcReceipt{Logs: logs[tx.Hash]}, bbh.Time, uint32(bbh.Confirmations)) if err != nil { - return nil, errors.Annotatef(err, "hash %v, height %v, txid %v", hash, height, tx.Hash.String()) + return nil, errors.Annotatef(err, "hash %v, height %v, txid %v", hash, height, tx.Hash) } btxs[i] = *btx + b.pendingTransactionsLock.Lock() + delete(b.pendingTransactions, tx.Hash) + b.pendingTransactionsLock.Unlock() } bbk := bchain.Block{ BlockHeader: *bbh, @@ -438,14 +494,38 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error // GetBlockInfo returns extended header (more info than in bchain.BlockHeader) with a list of txids func (b *EthereumRPC) GetBlockInfo(hash string) (*bchain.BlockInfo, error) { - // TODO - implement - return nil, errors.New("Not implemented yet") + raw, err := b.getBlockRaw(hash, 0, false) + if err != nil { + return nil, err + } + var head rpcHeader + var txs rpcBlockTxids + if err := json.Unmarshal(raw, &head); err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if err = json.Unmarshal(raw, &txs); err != nil { + return nil, err + } + bch, err := b.ethHeaderToBlockHeader(&head) + return &bchain.BlockInfo{ + BlockHeader: *bch, + Difficulty: json.Number(head.Difficulty), + Nonce: json.Number(head.Nonce), + Txids: txs.Transactions, + }, nil } // GetTransactionForMempool returns a transaction by the transaction ID. // It could be optimized for mempool, i.e. without block time and confirmations func (b *EthereumRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { - return b.GetTransaction(txid) + tx, err := b.GetTransaction(txid) + // it there is an error getting the tx or the tx is confirmed, remove it from pending transactions + if err != nil || (tx != nil && tx.Confirmations > 0) { + b.pendingTransactionsLock.Lock() + delete(b.pendingTransactions, txid) + b.pendingTransactionsLock.Unlock() + } + return tx, err } // GetTransaction returns a transaction by the transaction ID. @@ -453,32 +533,66 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() var tx *rpcTransaction - err := b.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", ethcommon.HexToHash(txid)) + hash := ethcommon.HexToHash(txid) + err := b.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) if err != nil { return nil, err } else if tx == nil { return nil, ethereum.NotFound - } else if tx.R == "" { - if !b.isETC { - return nil, errors.Annotatef(fmt.Errorf("server returned transaction without signature"), "txid %v", txid) - } else { - glog.Warning("server returned transaction without signature, txid ", txid) - } } var btx *bchain.Tx if tx.BlockNumber == "" { // mempool tx - btx, err = b.Parser.ethTxToTx(tx, 0, 0) + btx, err = b.Parser.ethTxToTx(tx, nil, 0, 0) if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } } else { - // non mempool tx - we must read the block header to get the block time - n, err := ethNumber(tx.BlockNumber) + // non mempool tx - read the block header to get the block time + raw, err := b.getBlockRaw(tx.BlockHash, 0, false) if err != nil { + return nil, err + } + var ht struct { + Time string `json:"timestamp"` + } + if err := json.Unmarshal(raw, &ht); err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + var time int64 + if time, err = ethNumber(ht.Time); err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } - h, err := b.client.HeaderByHash(ctx, *tx.BlockHash) + var receipt rpcReceipt + if b.isETC { + var rawReceipt json.RawMessage + var etcReceipt rpcEtcReceipt + err = b.rpc.CallContext(ctx, &rawReceipt, "eth_getTransactionReceipt", hash) + if err != nil { + return nil, errors.Annotatef(err, "txid %v", txid) + } + err = json.Unmarshal(rawReceipt, &etcReceipt) + if err == nil { + receipt.GasUsed = etcReceipt.GasUsed + receipt.Logs = etcReceipt.Logs + if etcReceipt.Status == 0 { + receipt.Status = "0x0" + } else { + receipt.Status = "0x1" + } + } else { + err = json.Unmarshal(rawReceipt, &receipt) + if err != nil { + return nil, errors.Annotatef(err, "unmarshal receipt for txid %v, %v", txid, string(rawReceipt)) + } + } + } else { + err = b.rpc.CallContext(ctx, &receipt, "eth_getTransactionReceipt", hash) + if err != nil { + return nil, errors.Annotatef(err, "txid %v", txid) + } + } + n, err := ethNumber(tx.BlockNumber) if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } @@ -486,7 +600,7 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } - btx, err = b.Parser.ethTxToTx(tx, h.Time.Int64(), confirmations) + btx, err = b.Parser.ethTxToTx(tx, &receipt, time, confirmations) if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } @@ -495,39 +609,47 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { } // GetTransactionSpecific returns json as returned by backend, with all coin specific data -func (b *EthereumRPC) GetTransactionSpecific(txid string) (json.RawMessage, error) { - ctx, cancel := context.WithTimeout(context.Background(), b.timeout) - defer cancel() - var tx json.RawMessage - err := b.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", ethcommon.HexToHash(txid)) - if err != nil { - return nil, err - } else if tx == nil { - return nil, ethereum.NotFound +func (b *EthereumRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, error) { + csd, ok := tx.CoinSpecificData.(completeTransaction) + if !ok { + ntx, err := b.GetTransaction(tx.Txid) + if err != nil { + return nil, err + } + csd, ok = ntx.CoinSpecificData.(completeTransaction) + if !ok { + return nil, errors.New("Cannot get CoinSpecificData") + } } - return tx, nil -} - -type rpcMempoolBlock struct { - Transactions []string `json:"transactions"` + m, err := json.Marshal(&csd) + return json.RawMessage(m), err } +// GetMempool returns transactions in mempool func (b *EthereumRPC) GetMempool() ([]string, error) { - ctx, cancel := context.WithTimeout(context.Background(), b.timeout) - defer cancel() - var raw json.RawMessage - var err error - err = b.rpc.CallContext(ctx, &raw, "eth_getBlockByNumber", "pending", false) + raw, err := b.getBlockRaw("pending", 0, false) if err != nil { return nil, err - } else if len(raw) == 0 { - return nil, bchain.ErrBlockNotFound } - var body rpcMempoolBlock - if err := json.Unmarshal(raw, &body); err != nil { - return nil, err + var body rpcBlockTxids + if len(raw) > 0 { + if err := json.Unmarshal(raw, &body); err != nil { + return nil, err + } } - return body.Transactions, nil + b.pendingTransactionsLock.Lock() + // join transactions returned by getBlockRaw with pendingTransactions from subscription + for _, txid := range body.Transactions { + b.pendingTransactions[txid] = struct{}{} + } + txids := make([]string, len(b.pendingTransactions)) + i := 0 + for txid := range b.pendingTransactions { + txids[i] = txid + i++ + } + b.pendingTransactionsLock.Unlock() + return txids, nil } // EstimateFee returns fee estimation @@ -539,21 +661,45 @@ func (b *EthereumRPC) EstimateFee(blocks int) (big.Int, error) { func (b *EthereumRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() - // TODO - what parameters of msg to use to get better estimate, maybe more data from the wallet are needed - a := ethcommon.HexToAddress("0x1234567890123456789012345678901234567890") - msg := ethereum.CallMsg{ - To: &a, - } - g, err := b.client.EstimateGas(ctx, msg) var r big.Int - if err != nil { - return r, err + gp, err := b.client.SuggestGasPrice(ctx) + if err == nil && b != nil { + r = *gp } - r.SetUint64(g) - return r, nil + return r, err } -// SendRawTransaction sends raw transaction. +func getStringFromMap(p string, params map[string]interface{}) (string, bool) { + v, ok := params[p] + if ok { + s, ok := v.(string) + return s, ok + } + return "", false +} + +// EthereumTypeEstimateGas returns estimation of gas consumption for given transaction parameters +func (b *EthereumRPC) EthereumTypeEstimateGas(params map[string]interface{}) (uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + msg := ethereum.CallMsg{} + s, ok := getStringFromMap("from", params) + if ok && len(s) > 0 { + msg.From = ethcommon.HexToAddress(s) + } + s, ok = getStringFromMap("to", params) + if ok && len(s) > 0 { + a := ethcommon.HexToAddress(s) + msg.To = &a + } + s, ok = getStringFromMap("data", params) + if ok && len(s) > 0 { + msg.Data = ethcommon.FromHex(s) + } + return b.client.EstimateGas(ctx, msg) +} + +// SendRawTransaction sends raw transaction func (b *EthereumRPC) SendRawTransaction(hex string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() @@ -574,24 +720,43 @@ func (b *EthereumRPC) SendRawTransaction(hex string) (string, error) { return result, nil } +// EthereumTypeGetBalance returns current balance of an address +func (b *EthereumRPC) EthereumTypeGetBalance(addrDesc bchain.AddressDescriptor) (*big.Int, error) { + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + return b.client.BalanceAt(ctx, ethcommon.BytesToAddress(addrDesc), nil) +} + +// EthereumTypeGetNonce returns current balance of an address +func (b *EthereumRPC) EthereumTypeGetNonce(addrDesc bchain.AddressDescriptor) (uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + defer cancel() + return b.client.NonceAt(ctx, ethcommon.BytesToAddress(addrDesc), nil) +} + +// ResyncMempool gets mempool transactions and maps output scripts to transactions. +// ResyncMempool is not reentrant, it should be called from a single thread. +// Return value is number of transactions in mempool func (b *EthereumRPC) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (int, error) { return b.Mempool.Resync(onNewTxAddr) } // GetMempoolTransactions returns slice of mempool transactions for given address -func (b *EthereumRPC) GetMempoolTransactions(address string) ([]string, error) { +func (b *EthereumRPC) GetMempoolTransactions(address string) ([]bchain.Outpoint, error) { return b.Mempool.GetTransactions(address) } // GetMempoolTransactionsForAddrDesc returns slice of mempool transactions for given address descriptor -func (b *EthereumRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]string, error) { +func (b *EthereumRPC) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) ([]bchain.Outpoint, error) { return b.Mempool.GetAddrDescTransactions(addrDesc) } +// GetMempoolEntry is not supported by etherem func (b *EthereumRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { - return nil, errors.New("GetMempoolEntry: not implemented") + return nil, errors.New("GetMempoolEntry: not supported") } +// GetChainParser returns ethereum BlockChainParser func (b *EthereumRPC) GetChainParser() bchain.BlockChainParser { return b.Parser } diff --git a/bchain/coins/eth/tx.pb.go b/bchain/coins/eth/tx.pb.go index 17b0860c..ff10c6cb 100644 --- a/bchain/coins/eth/tx.pb.go +++ b/bchain/coins/eth/tx.pb.go @@ -8,7 +8,7 @@ It is generated from these files: tx.proto It has these top-level messages: - ProtoTransaction + ProtoCompleteTransaction */ package eth @@ -27,149 +27,234 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type ProtoTransaction struct { - AccountNonce uint64 `protobuf:"varint,1,opt,name=AccountNonce" json:"AccountNonce,omitempty"` - Price []byte `protobuf:"bytes,2,opt,name=Price,proto3" json:"Price,omitempty"` - GasLimit uint64 `protobuf:"varint,3,opt,name=GasLimit" json:"GasLimit,omitempty"` - Value []byte `protobuf:"bytes,4,opt,name=Value,proto3" json:"Value,omitempty"` - Payload []byte `protobuf:"bytes,5,opt,name=Payload,proto3" json:"Payload,omitempty"` - Hash []byte `protobuf:"bytes,6,opt,name=Hash,proto3" json:"Hash,omitempty"` - BlockNumber uint32 `protobuf:"varint,7,opt,name=BlockNumber" json:"BlockNumber,omitempty"` - BlockTime uint64 `protobuf:"varint,8,opt,name=BlockTime" json:"BlockTime,omitempty"` - To []byte `protobuf:"bytes,9,opt,name=To,proto3" json:"To,omitempty"` - From []byte `protobuf:"bytes,10,opt,name=From,proto3" json:"From,omitempty"` - TransactionIndex uint32 `protobuf:"varint,11,opt,name=TransactionIndex" json:"TransactionIndex,omitempty"` - V []byte `protobuf:"bytes,12,opt,name=V,proto3" json:"V,omitempty"` - R []byte `protobuf:"bytes,13,opt,name=R,proto3" json:"R,omitempty"` - S []byte `protobuf:"bytes,14,opt,name=S,proto3" json:"S,omitempty"` +type ProtoCompleteTransaction struct { + BlockNumber uint32 `protobuf:"varint,1,opt,name=BlockNumber" json:"BlockNumber,omitempty"` + BlockTime uint64 `protobuf:"varint,2,opt,name=BlockTime" json:"BlockTime,omitempty"` + Tx *ProtoCompleteTransaction_TxType `protobuf:"bytes,3,opt,name=Tx" json:"Tx,omitempty"` + Receipt *ProtoCompleteTransaction_ReceiptType `protobuf:"bytes,4,opt,name=Receipt" json:"Receipt,omitempty"` } -func (m *ProtoTransaction) Reset() { *m = ProtoTransaction{} } -func (m *ProtoTransaction) String() string { return proto.CompactTextString(m) } -func (*ProtoTransaction) ProtoMessage() {} -func (*ProtoTransaction) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (m *ProtoCompleteTransaction) Reset() { *m = ProtoCompleteTransaction{} } +func (m *ProtoCompleteTransaction) String() string { return proto.CompactTextString(m) } +func (*ProtoCompleteTransaction) ProtoMessage() {} +func (*ProtoCompleteTransaction) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } -func (m *ProtoTransaction) GetAccountNonce() uint64 { - if m != nil { - return m.AccountNonce - } - return 0 -} - -func (m *ProtoTransaction) GetPrice() []byte { - if m != nil { - return m.Price - } - return nil -} - -func (m *ProtoTransaction) GetGasLimit() uint64 { - if m != nil { - return m.GasLimit - } - return 0 -} - -func (m *ProtoTransaction) GetValue() []byte { - if m != nil { - return m.Value - } - return nil -} - -func (m *ProtoTransaction) GetPayload() []byte { - if m != nil { - return m.Payload - } - return nil -} - -func (m *ProtoTransaction) GetHash() []byte { - if m != nil { - return m.Hash - } - return nil -} - -func (m *ProtoTransaction) GetBlockNumber() uint32 { +func (m *ProtoCompleteTransaction) GetBlockNumber() uint32 { if m != nil { return m.BlockNumber } return 0 } -func (m *ProtoTransaction) GetBlockTime() uint64 { +func (m *ProtoCompleteTransaction) GetBlockTime() uint64 { if m != nil { return m.BlockTime } return 0 } -func (m *ProtoTransaction) GetTo() []byte { +func (m *ProtoCompleteTransaction) GetTx() *ProtoCompleteTransaction_TxType { + if m != nil { + return m.Tx + } + return nil +} + +func (m *ProtoCompleteTransaction) GetReceipt() *ProtoCompleteTransaction_ReceiptType { + if m != nil { + return m.Receipt + } + return nil +} + +type ProtoCompleteTransaction_TxType struct { + AccountNonce uint64 `protobuf:"varint,1,opt,name=AccountNonce" json:"AccountNonce,omitempty"` + GasPrice []byte `protobuf:"bytes,2,opt,name=GasPrice,proto3" json:"GasPrice,omitempty"` + GasLimit uint64 `protobuf:"varint,3,opt,name=GasLimit" json:"GasLimit,omitempty"` + Value []byte `protobuf:"bytes,4,opt,name=Value,proto3" json:"Value,omitempty"` + Payload []byte `protobuf:"bytes,5,opt,name=Payload,proto3" json:"Payload,omitempty"` + Hash []byte `protobuf:"bytes,6,opt,name=Hash,proto3" json:"Hash,omitempty"` + To []byte `protobuf:"bytes,7,opt,name=To,proto3" json:"To,omitempty"` + From []byte `protobuf:"bytes,8,opt,name=From,proto3" json:"From,omitempty"` + TransactionIndex uint32 `protobuf:"varint,9,opt,name=TransactionIndex" json:"TransactionIndex,omitempty"` +} + +func (m *ProtoCompleteTransaction_TxType) Reset() { *m = ProtoCompleteTransaction_TxType{} } +func (m *ProtoCompleteTransaction_TxType) String() string { return proto.CompactTextString(m) } +func (*ProtoCompleteTransaction_TxType) ProtoMessage() {} +func (*ProtoCompleteTransaction_TxType) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{0, 0} +} + +func (m *ProtoCompleteTransaction_TxType) GetAccountNonce() uint64 { + if m != nil { + return m.AccountNonce + } + return 0 +} + +func (m *ProtoCompleteTransaction_TxType) GetGasPrice() []byte { + if m != nil { + return m.GasPrice + } + return nil +} + +func (m *ProtoCompleteTransaction_TxType) GetGasLimit() uint64 { + if m != nil { + return m.GasLimit + } + return 0 +} + +func (m *ProtoCompleteTransaction_TxType) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +func (m *ProtoCompleteTransaction_TxType) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func (m *ProtoCompleteTransaction_TxType) GetHash() []byte { + if m != nil { + return m.Hash + } + return nil +} + +func (m *ProtoCompleteTransaction_TxType) GetTo() []byte { if m != nil { return m.To } return nil } -func (m *ProtoTransaction) GetFrom() []byte { +func (m *ProtoCompleteTransaction_TxType) GetFrom() []byte { if m != nil { return m.From } return nil } -func (m *ProtoTransaction) GetTransactionIndex() uint32 { +func (m *ProtoCompleteTransaction_TxType) GetTransactionIndex() uint32 { if m != nil { return m.TransactionIndex } return 0 } -func (m *ProtoTransaction) GetV() []byte { +type ProtoCompleteTransaction_ReceiptType struct { + GasUsed []byte `protobuf:"bytes,1,opt,name=GasUsed,proto3" json:"GasUsed,omitempty"` + Status []byte `protobuf:"bytes,2,opt,name=Status,proto3" json:"Status,omitempty"` + Log []*ProtoCompleteTransaction_ReceiptType_LogType `protobuf:"bytes,3,rep,name=Log" json:"Log,omitempty"` +} + +func (m *ProtoCompleteTransaction_ReceiptType) Reset() { *m = ProtoCompleteTransaction_ReceiptType{} } +func (m *ProtoCompleteTransaction_ReceiptType) String() string { return proto.CompactTextString(m) } +func (*ProtoCompleteTransaction_ReceiptType) ProtoMessage() {} +func (*ProtoCompleteTransaction_ReceiptType) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{0, 1} +} + +func (m *ProtoCompleteTransaction_ReceiptType) GetGasUsed() []byte { if m != nil { - return m.V + return m.GasUsed } return nil } -func (m *ProtoTransaction) GetR() []byte { +func (m *ProtoCompleteTransaction_ReceiptType) GetStatus() []byte { if m != nil { - return m.R + return m.Status } return nil } -func (m *ProtoTransaction) GetS() []byte { +func (m *ProtoCompleteTransaction_ReceiptType) GetLog() []*ProtoCompleteTransaction_ReceiptType_LogType { if m != nil { - return m.S + return m.Log + } + return nil +} + +type ProtoCompleteTransaction_ReceiptType_LogType struct { + Address []byte `protobuf:"bytes,1,opt,name=Address,proto3" json:"Address,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` + Topics [][]byte `protobuf:"bytes,3,rep,name=Topics,proto3" json:"Topics,omitempty"` +} + +func (m *ProtoCompleteTransaction_ReceiptType_LogType) Reset() { + *m = ProtoCompleteTransaction_ReceiptType_LogType{} +} +func (m *ProtoCompleteTransaction_ReceiptType_LogType) String() string { + return proto.CompactTextString(m) +} +func (*ProtoCompleteTransaction_ReceiptType_LogType) ProtoMessage() {} +func (*ProtoCompleteTransaction_ReceiptType_LogType) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{0, 1, 0} +} + +func (m *ProtoCompleteTransaction_ReceiptType_LogType) GetAddress() []byte { + if m != nil { + return m.Address + } + return nil +} + +func (m *ProtoCompleteTransaction_ReceiptType_LogType) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *ProtoCompleteTransaction_ReceiptType_LogType) GetTopics() [][]byte { + if m != nil { + return m.Topics } return nil } func init() { - proto.RegisterType((*ProtoTransaction)(nil), "eth.ProtoTransaction") + proto.RegisterType((*ProtoCompleteTransaction)(nil), "eth.ProtoCompleteTransaction") + proto.RegisterType((*ProtoCompleteTransaction_TxType)(nil), "eth.ProtoCompleteTransaction.TxType") + proto.RegisterType((*ProtoCompleteTransaction_ReceiptType)(nil), "eth.ProtoCompleteTransaction.ReceiptType") + proto.RegisterType((*ProtoCompleteTransaction_ReceiptType_LogType)(nil), "eth.ProtoCompleteTransaction.ReceiptType.LogType") } func init() { proto.RegisterFile("tx.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 262 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0xbd, 0x6a, 0xeb, 0x40, - 0x10, 0x85, 0x59, 0x59, 0xb6, 0xe5, 0xb1, 0x6c, 0xcc, 0x70, 0x8b, 0xe1, 0x92, 0x42, 0xb8, 0x12, - 0x29, 0xd2, 0xe4, 0x09, 0x92, 0x22, 0x3f, 0x10, 0x8c, 0x90, 0x85, 0xfa, 0xf5, 0x7a, 0xc1, 0x22, - 0x92, 0x26, 0x48, 0x2b, 0x70, 0x5e, 0x38, 0xcf, 0x11, 0x76, 0x44, 0x12, 0x87, 0x74, 0xf3, 0x7d, - 0x70, 0xf6, 0x2c, 0x07, 0x22, 0x77, 0xbe, 0x79, 0xeb, 0xd8, 0x31, 0x4e, 0xac, 0x3b, 0x6d, 0x3f, - 0x02, 0xd8, 0x64, 0x1e, 0x8b, 0x4e, 0xb7, 0xbd, 0x36, 0xae, 0xe2, 0x16, 0xb7, 0x10, 0xdf, 0x19, - 0xc3, 0x43, 0xeb, 0x76, 0xdc, 0x1a, 0x4b, 0x2a, 0x51, 0x69, 0x98, 0xff, 0x72, 0xf8, 0x0f, 0xa6, - 0x59, 0x57, 0x19, 0x4b, 0x41, 0xa2, 0xd2, 0x38, 0x1f, 0x01, 0xff, 0x43, 0xf4, 0xa8, 0xfb, 0x97, - 0xaa, 0xa9, 0x1c, 0x4d, 0x24, 0xf5, 0xcd, 0x3e, 0x51, 0xea, 0x7a, 0xb0, 0x14, 0x8e, 0x09, 0x01, - 0x24, 0x98, 0x67, 0xfa, 0xbd, 0x66, 0x7d, 0xa4, 0xa9, 0xf8, 0x2f, 0x44, 0x84, 0xf0, 0x49, 0xf7, - 0x27, 0x9a, 0x89, 0x96, 0x1b, 0x13, 0x58, 0xde, 0xd7, 0x6c, 0x5e, 0x77, 0x43, 0x73, 0xb0, 0x1d, - 0xcd, 0x13, 0x95, 0xae, 0xf2, 0x4b, 0x85, 0x57, 0xb0, 0x10, 0x2c, 0xaa, 0xc6, 0x52, 0x24, 0x5f, - 0xf8, 0x11, 0xb8, 0x86, 0xa0, 0x60, 0x5a, 0xc8, 0x8b, 0x41, 0xc1, 0xbe, 0xe3, 0xa1, 0xe3, 0x86, - 0x60, 0xec, 0xf0, 0x37, 0x5e, 0xc3, 0xe6, 0x62, 0x8c, 0xe7, 0xf6, 0x68, 0xcf, 0xb4, 0x94, 0xa2, - 0x3f, 0x1e, 0x63, 0x50, 0x25, 0xc5, 0x12, 0x56, 0xa5, 0xa7, 0x9c, 0x56, 0x23, 0xe5, 0x9e, 0xf6, - 0xb4, 0x1e, 0x69, 0x7f, 0x98, 0xc9, 0xe8, 0xb7, 0x9f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x15, 0xc8, - 0xe4, 0x30, 0x80, 0x01, 0x00, 0x00, + // 393 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xdf, 0x8a, 0xd4, 0x30, + 0x14, 0xc6, 0xe9, 0x9f, 0xf9, 0xb3, 0xa7, 0x55, 0x24, 0x88, 0x84, 0xe2, 0x45, 0x59, 0xbc, 0xa8, + 0x5e, 0x14, 0x5c, 0x7d, 0x81, 0x75, 0xc4, 0x55, 0x18, 0xd6, 0x21, 0x46, 0xef, 0xb3, 0x69, 0xd8, + 0x29, 0xb6, 0x4d, 0x69, 0x52, 0xe8, 0xbe, 0x91, 0x2f, 0xe4, 0xbb, 0x78, 0x29, 0x39, 0x4d, 0xd7, + 0x11, 0x51, 0xbc, 0x3b, 0xbf, 0x6f, 0xce, 0x37, 0xf9, 0xbe, 0xa4, 0xb0, 0xb5, 0x53, 0xd9, 0x0f, + 0xda, 0x6a, 0x12, 0x29, 0x7b, 0x3c, 0xff, 0xb6, 0x02, 0x7a, 0x70, 0xb8, 0xd3, 0x6d, 0xdf, 0x28, + 0xab, 0xf8, 0x20, 0x3a, 0x23, 0xa4, 0xad, 0x75, 0x47, 0x72, 0x48, 0xde, 0x34, 0x5a, 0x7e, 0xbd, + 0x1e, 0xdb, 0x1b, 0x35, 0xd0, 0x20, 0x0f, 0x8a, 0x07, 0xec, 0x54, 0x22, 0x4f, 0xe1, 0x0c, 0x91, + 0xd7, 0xad, 0xa2, 0x61, 0x1e, 0x14, 0x31, 0xfb, 0x25, 0x90, 0xd7, 0x10, 0xf2, 0x89, 0x46, 0x79, + 0x50, 0x24, 0x17, 0xcf, 0x4a, 0x65, 0x8f, 0xe5, 0xdf, 0x8e, 0x2a, 0xf9, 0xc4, 0xef, 0x7a, 0xc5, + 0x42, 0x3e, 0x91, 0x1d, 0x6c, 0x98, 0x92, 0xaa, 0xee, 0x2d, 0x8d, 0xd1, 0xfa, 0xfc, 0xdf, 0x56, + 0xbf, 0x8c, 0xfe, 0xc5, 0x99, 0xfd, 0x08, 0x60, 0x3d, 0xff, 0x27, 0x39, 0x87, 0xf4, 0x52, 0x4a, + 0x3d, 0x76, 0xf6, 0x5a, 0x77, 0x52, 0x61, 0x8d, 0x98, 0xfd, 0xa6, 0x91, 0x0c, 0xb6, 0x57, 0xc2, + 0x1c, 0x86, 0x5a, 0xce, 0x35, 0x52, 0x76, 0xcf, 0xfe, 0xb7, 0x7d, 0xdd, 0xd6, 0x16, 0xbb, 0xc4, + 0xec, 0x9e, 0xc9, 0x63, 0x58, 0x7d, 0x11, 0xcd, 0xa8, 0x30, 0x69, 0xca, 0x66, 0x20, 0x14, 0x36, + 0x07, 0x71, 0xd7, 0x68, 0x51, 0xd1, 0x15, 0xea, 0x0b, 0x12, 0x02, 0xf1, 0x7b, 0x61, 0x8e, 0x74, + 0x8d, 0x32, 0xce, 0xe4, 0x21, 0x84, 0x5c, 0xd3, 0x0d, 0x2a, 0x21, 0xd7, 0x6e, 0xe7, 0xdd, 0xa0, + 0x5b, 0xba, 0x9d, 0x77, 0xdc, 0x4c, 0x5e, 0xc0, 0xa3, 0x93, 0xca, 0x1f, 0xba, 0x4a, 0x4d, 0xf4, + 0x0c, 0x9f, 0xe3, 0x0f, 0x3d, 0xfb, 0x1e, 0x40, 0x72, 0x72, 0x27, 0x2e, 0xcd, 0x95, 0x30, 0x9f, + 0x8d, 0xaa, 0xb0, 0x7a, 0xca, 0x16, 0x24, 0x4f, 0x60, 0xfd, 0xc9, 0x0a, 0x3b, 0x1a, 0xdf, 0xd9, + 0x13, 0xd9, 0x41, 0xb4, 0xd7, 0xb7, 0x34, 0xca, 0xa3, 0x22, 0xb9, 0x78, 0xf9, 0xdf, 0xb7, 0x5f, + 0xee, 0xf5, 0x2d, 0xbe, 0x82, 0x73, 0x67, 0x1f, 0x61, 0xe3, 0xd9, 0x25, 0xb8, 0xac, 0xaa, 0x41, + 0x19, 0xb3, 0x24, 0xf0, 0xe8, 0xba, 0xbe, 0x15, 0x56, 0xf8, 0xf3, 0x71, 0x76, 0xa9, 0xb8, 0xee, + 0x6b, 0x69, 0x30, 0x40, 0xca, 0x3c, 0xdd, 0xac, 0xf1, 0xb3, 0x7d, 0xf5, 0x33, 0x00, 0x00, 0xff, + 0xff, 0xde, 0xd5, 0x28, 0xa3, 0xc2, 0x02, 0x00, 0x00, } diff --git a/bchain/coins/eth/tx.proto b/bchain/coins/eth/tx.proto index c7ed0fa9..ef7c4ce0 100644 --- a/bchain/coins/eth/tx.proto +++ b/bchain/coins/eth/tx.proto @@ -1,19 +1,30 @@ syntax = "proto3"; package eth; - message ProtoTransaction { - uint64 AccountNonce = 1; - bytes Price = 2; - uint64 GasLimit = 3; - bytes Value = 4; - bytes Payload = 5; - bytes Hash = 6; - uint32 BlockNumber = 7; - uint64 BlockTime = 8; - bytes To = 9; - bytes From = 10; - uint32 TransactionIndex = 11; - bytes V = 12; - bytes R = 13; - bytes S = 14; + message ProtoCompleteTransaction { + message TxType { + uint64 AccountNonce = 1; + bytes GasPrice = 2; + uint64 GasLimit = 3; + bytes Value = 4; + bytes Payload = 5; + bytes Hash = 6; + bytes To = 7; + bytes From = 8; + uint32 TransactionIndex = 9; + } + message ReceiptType { + message LogType { + bytes Address = 1; + bytes Data = 2; + repeated bytes Topics = 3; + } + bytes GasUsed = 1; + bytes Status = 2; + repeated LogType Log = 3; + } + uint32 BlockNumber = 1; + uint64 BlockTime = 2; + TxType Tx = 3; + ReceiptType Receipt = 4; } \ No newline at end of file diff --git a/bchain/coins/grs/grsparser_test.go b/bchain/coins/grs/grsparser_test.go index 5b40ec38..70e2ed3f 100644 --- a/bchain/coins/grs/grsparser_test.go +++ b/bchain/coins/grs/grsparser_test.go @@ -18,8 +18,8 @@ import ( var ( testTx1, testTx2 bchain.Tx - testTxPacked1 = "0a20f56521b17b828897f72b30dd21b0192fd942342e89acbb06abf1d446282c30f512bf0101000000014a9d1fdba915e0907ab02f04f88898863112a2b4fdcf872c7414588c47c874cb000000006a47304402201fb96d20d0778f54520ab59afe70d5fb20e500ecc9f02281cf57934e8029e8e10220383d5a3e80f2e1eb92765b6da0f23d454aecbd8236f083d483e9a7430236876101210331693756f749180aeed0a65a0fab0625a2250bd9abca502282a4cf0723152e67ffffffff01a0330300000000001976a914fe40329c95c5598ac60752a5310b320cb52d18e688ac0000000018ffff87da05200028a6f383013298010a001220cb74c8478c5814742c87cffdb4a21231869888f8042fb07a90e015a9db1f9d4a1800226a47304402201fb96d20d0778f54520ab59afe70d5fb20e500ecc9f02281cf57934e8029e8e10220383d5a3e80f2e1eb92765b6da0f23d454aecbd8236f083d483e9a7430236876101210331693756f749180aeed0a65a0fab0625a2250bd9abca502282a4cf0723152e6728ffffffff0f3a460a030333a010001a1976a914fe40329c95c5598ac60752a5310b320cb52d18e688ac222246744d347a416e39615659674867786d616d5742675750795a7362365268766b4139" - testTxPacked2 = "0a209b5c4859a8a31e69788cb4402812bb28f14ad71cbd8c60b09903478bc56f79a312e00101000000000101d1613f483f2086d076c82fe34674385a86beb08f052d5405fe1aed397f852f4f0000000000feffffff02404b4c000000000017a9147a55d61848e77ca266e79a39bfc85c580a6426c987a8386f0000000000160014cc8067093f6f843d6d3e22004a4290cd0c0f336b02483045022100ea8780bc1e60e14e945a80654a41748bbf1aa7d6f2e40a88d91dfc2de1f34bd10220181a474a3420444bd188501d8d270736e1e9fe379da9970de992ff445b0972e3012103adc58245cf28406af0ef5cc24b8afba7f1be6c72f279b642d85c48798685f862d9ed090018caa384da0520d9db2728dadb27322c0a0012204f2f857f39ed1afe05542d058fb0be865a387446e32fc876d086203f483f61d1180028feffffff0f3a450a034c4b4010001a17a9147a55d61848e77ca266e79a39bfc85c580a6426c9872223324e345135466855323439374272794666556762716b414a453837614b4476335633653a4d0a036f38a810011a160014cc8067093f6f843d6d3e22004a4290cd0c0f336b222c746772733171656a7178777a666c64377a72366d663779677179357335736535787137766d74396c6b643537" + testTxPacked1 = "0a20f56521b17b828897f72b30dd21b0192fd942342e89acbb06abf1d446282c30f512bf0101000000014a9d1fdba915e0907ab02f04f88898863112a2b4fdcf872c7414588c47c874cb000000006a47304402201fb96d20d0778f54520ab59afe70d5fb20e500ecc9f02281cf57934e8029e8e10220383d5a3e80f2e1eb92765b6da0f23d454aecbd8236f083d483e9a7430236876101210331693756f749180aeed0a65a0fab0625a2250bd9abca502282a4cf0723152e67ffffffff01a0330300000000001976a914fe40329c95c5598ac60752a5310b320cb52d18e688ac0000000018ffff87da05200028a6f383013298010a001220cb74c8478c5814742c87cffdb4a21231869888f8042fb07a90e015a9db1f9d4a1800226a47304402201fb96d20d0778f54520ab59afe70d5fb20e500ecc9f02281cf57934e8029e8e10220383d5a3e80f2e1eb92765b6da0f23d454aecbd8236f083d483e9a7430236876101210331693756f749180aeed0a65a0fab0625a2250bd9abca502282a4cf0723152e6728ffffffff0f3a460a030333a010001a1976a914fe40329c95c5598ac60752a5310b320cb52d18e688ac222246744d347a416e39615659674867786d616d5742675750795a7362365268766b41394000" + testTxPacked2 = "0a209b5c4859a8a31e69788cb4402812bb28f14ad71cbd8c60b09903478bc56f79a312e00101000000000101d1613f483f2086d076c82fe34674385a86beb08f052d5405fe1aed397f852f4f0000000000feffffff02404b4c000000000017a9147a55d61848e77ca266e79a39bfc85c580a6426c987a8386f0000000000160014cc8067093f6f843d6d3e22004a4290cd0c0f336b02483045022100ea8780bc1e60e14e945a80654a41748bbf1aa7d6f2e40a88d91dfc2de1f34bd10220181a474a3420444bd188501d8d270736e1e9fe379da9970de992ff445b0972e3012103adc58245cf28406af0ef5cc24b8afba7f1be6c72f279b642d85c48798685f862d9ed090018caa384da0520d9db2728dadb27322c0a0012204f2f857f39ed1afe05542d058fb0be865a387446e32fc876d086203f483f61d1180028feffffff0f3a450a034c4b4010001a17a9147a55d61848e77ca266e79a39bfc85c580a6426c9872223324e345135466855323439374272794666556762716b414a453837614b4476335633653a4d0a036f38a810011a160014cc8067093f6f843d6d3e22004a4290cd0c0f336b222c746772733171656a7178777a666c64377a72366d663779677179357335736535787137766d74396c6b6435374000" ) func init() { diff --git a/bchain/coins/liquid/liquidparser.go b/bchain/coins/liquid/liquidparser.go new file mode 100644 index 00000000..53997c78 --- /dev/null +++ b/bchain/coins/liquid/liquidparser.go @@ -0,0 +1,64 @@ +package liquid + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + + "github.com/btcsuite/btcd/wire" + "github.com/jakm/btcutil/chaincfg" +) + +const ( + MainnetMagic wire.BitcoinNet = 0xdab5bffa +) + +var ( + MainNetParams chaincfg.Params +) + +func init() { + MainNetParams = chaincfg.MainNetParams + MainNetParams.Net = MainnetMagic + MainNetParams.PubKeyHashAddrID = []byte{57} + MainNetParams.ScriptHashAddrID = []byte{39} + // BLINDED_ADDRESS 12 +} + +// LiquidParser handle +type LiquidParser struct { + *btc.BitcoinParser + baseparser *bchain.BaseParser +} + +// NewLiquidParser returns new LiquidParser instance +func NewLiquidParser(params *chaincfg.Params, c *btc.Configuration) *LiquidParser { + return &LiquidParser{ + BitcoinParser: btc.NewBitcoinParser(params, c), + baseparser: &bchain.BaseParser{}, + } +} + +// GetChainParams contains network parameters for the main GameCredits network, +// and the test GameCredits network +func GetChainParams(chain string) *chaincfg.Params { + if !chaincfg.IsRegistered(&MainNetParams) { + err := chaincfg.Register(&MainNetParams) + if err != nil { + panic(err) + } + } + switch chain { + default: + return &MainNetParams + } +} + +// PackTx packs transaction to byte array using protobuf +func (p *LiquidParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { + return p.baseparser.PackTx(tx, height, blockTime) +} + +// UnpackTx unpacks transaction from protobuf byte array +func (p *LiquidParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { + return p.baseparser.UnpackTx(buf) +} diff --git a/bchain/coins/liquid/liquidparser_test.go b/bchain/coins/liquid/liquidparser_test.go new file mode 100644 index 00000000..ced53328 --- /dev/null +++ b/bchain/coins/liquid/liquidparser_test.go @@ -0,0 +1,268 @@ +// build unittest + +package liquid + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "encoding/hex" + "math/big" + "os" + "reflect" + "testing" + + "github.com/jakm/btcutil/chaincfg" +) + +func TestMain(m *testing.M) { + c := m.Run() + chaincfg.ResetParams() + os.Exit(c) +} + +func Test_GetAddrDescFromAddress(t *testing.T) { + type args struct { + address string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "P2PKH1", + args: args{address: "QHU1yszeZwVeuJosGJ4JDHuKaLRWmdEYDF"}, + want: "76a914dd95db91e8f914cbd63bae8e307d54399f060cd688ac", + wantErr: false, + }, + { + name: "P2PKH2", + args: args{address: "PwogNb9zqhrwPneDqVp18GqcVuygsCvXkU"}, + want: "76a91405eb4afe4615751cfb813a00846a8d9ef8a9a2e588ac", + wantErr: false, + }, + { + name: "P2SH1", + args: args{address: "GhWTZqLPHRK8KfuT6yo1wGisQzn4cXrbPP"}, + want: "a9140394b3cf9a44782c10105b93962daa8dba304d7f87", + wantErr: false, + }, + } + parser := NewLiquidParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parser.GetAddrDescFromAddress(tt.args.address) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddrDescFromAddress() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("GetAddrDescFromAddress() = %v, want %v", h, tt.want) + } + }) + } +} + +func Test_GetAddressesFromAddrDesc(t *testing.T) { + type args struct { + script string + } + tests := []struct { + name string + args args + want []string + want2 bool + wantErr bool + }{ + { + name: "P2PKH1", + args: args{script: "76a914dd95db91e8f914cbd63bae8e307d54399f060cd688ac"}, + want: []string{"QHU1yszeZwVeuJosGJ4JDHuKaLRWmdEYDF"}, + want2: true, + wantErr: false, + }, + { + name: "P2PKH2", + args: args{script: "76a91405eb4afe4615751cfb813a00846a8d9ef8a9a2e588ac"}, + want: []string{"PwogNb9zqhrwPneDqVp18GqcVuygsCvXkU"}, + want2: true, + wantErr: false, + }, + { + name: "P2SH", + args: args{script: "a9140394b3cf9a44782c10105b93962daa8dba304d7f87"}, + want: []string{"GhWTZqLPHRK8KfuT6yo1wGisQzn4cXrbPP"}, + want2: true, + wantErr: false, + }, + { + name: "P2PK compressed", + args: args{script: "21020e46e79a2a8d12b9b5d12c7a91adb4e454edfae43c0a0cb805427d2ac7613fd9ac"}, + want: []string{"QKKEbCNAV7BYdtfSb3cQcJ7QSmFMvtXETz"}, + want2: false, + wantErr: false, + }, + { + name: "P2PK uncompressed", + args: args{script: "41041057356b91bfd3efeff5fc0fa8b865faafafb67bd653c5da2cd16ce15c7b86db0e622c8e1e135f68918a23601eb49208c1ac72c7b64a4ee99c396cf788da16ccac"}, + want: []string{"QDoUiWY7iZXDrkBzXdk6dru8DbvGqExXuf"}, + want2: false, + wantErr: false, + }, + } + + parser := NewLiquidParser(GetChainParams("main"), &btc.Configuration{}) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.script) + got, got2, err := parser.GetAddressesFromAddrDesc(b) + if (err != nil) != tt.wantErr { + t.Errorf("GetAddressesFromAddrDesc() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got2, tt.want2) { + t.Errorf("GetAddressesFromAddrDesc() = %v, want %v", got2, tt.want2) + } + }) + } +} + +var ( + testTx1 bchain.Tx + + testTxPacked1 = "0a207aa1af9481f2d744c96015b1baea6ba753790971ff265adfd93775de0234bfd612b51a020000000101a99547d213b005f355da348de54f5eb370fbc6a5687e412897ef0ec4ce237d75020000006b483045022100ae926c96c746308e7488e022f4ad1db94d5d0c8683f6fa6ded3afb13d8e20578022074aa8ebfe20adaf25beed70c60cfb5007278bec17875f11f7a5b3c86eb60a96901210391abdcd113c40b56f13a548e8624d6ad8d7a162b33ef81020d046cadbda26637feffffff030af62b535fc393152f6d575708b271e3a53514cdcf65508c7d12e8fd06709ce24208e192e1315aa94c3bc0eddbb0ae195aff0a1ba2e19773b79e5d1b29fdd8df211b02f8b255b83fa13f40745fb5054b89dc7dcba497c850086a8726bac66ac22d4be31976a914d79c1c8b67a0275c60e33b67bbd0e19a79b9276388ac016d521c38ec1ea15734ae22b7c46064412829c0d0579f0a713d1c04ede979026f01000000000000000000256a23426c6f636b73747265616d2e696e666f206973206120636f6f6c206578706c6f726572016d521c38ec1ea15734ae22b7c46064412829c0d0579f0a713d1c04ede979026f0100000000000004800000000000000000000043010001db986007e38ccb7bdef1661fcf633cbde3d8850cd116736d9b0b0b027901921694954b26036eb51d9424c38083aa3d937998ea81077843925513523eb6ec3337fd4d0b602300000000000000011486003b49c8ab1a6aa09a8eaec4ed83dbda4cb4ef9651114513823a6eefb1dbf971631d88a229a3b027a94f9f15966765f6f5ce3245057c57aedf8f4c69e355f4264a10e2c7dad691879844074ae5e37152c828aba89035e928c63b1c1590638333d05ed08442538ebf7dff9c94ce11ab6bb6d5c9535fce99fab023aabc7215b52eed15454333428b065fb5d8b03aa2fe1de5004f9d8fca717bcd682f1f9caa6561bdafbe69c8423166f7e33867f0bc4fdd85224ff1533839762530d47a1f053c40c7e38f84a1431ad03398bc9d634384aec0f22e75ac4a94e3703ec8715d0791564a8b1509eab4bf7543ef2e5c20fe9fb6c2a85afe42fcdba64c628103ea5693287a28ded79f517a3e877fe287ca6f3d1229295abaf5c1b3b43ab54ec4697c941da47e0934718b697a414d8fd1a722eb23ceb554afb1b4c807a94507a35b153f19da10c6688c71efa0d1ad15c8ac2f3d786ae8c43136cacf333c2643e521322314346b201427f1ac975340cf3caf102245d8cf45709bb51e41fd357f1fb7316138f1992e5646842eb4fe18ef6a7096f7e4c1e69950285f30c64dc1da661c055944356f140d6298f5eaf733799bef034a8afb05f6f74239e572acf5d1c2f9e028b806caab13eb19148456b3db0f719ad76818eaea11e7989e74d858743ffba738a5f5c7d12ab1b049c594656821620e20817c6acc5b3d3725f9183d3c01a0190d5d991d1603682418f4a9c55b59cab8787f463123e1efd7d3e7f75dfb9ccf931ae2dc965b82cabf2af6b93293a4b00d7145d97a24679076157bade5ba4d7577922052c719ebf2493f5e9d0cf8e6f82666e8732dd91068395e528d0ea532e27ae8a84ed516dabd30dd990a1539d93f7b775039d53f1a12d7e7747a64f6aca9d7589330c8adf8854dce0fee0be212716d6c1b075fabb9d0c815e613c772f7c57e7efdb4e64a5ef47c9cf384deab55191d7b4b0cb27986649eca2c4286a01a7d3c304f008c6d3fe53abf320afc7602654cf4b69b87fd2b9b6794fe7cc77e92bbef572dc8ddb1530da6d82268cd32db112afb8c2db69651959c1ad39d804178cae05f856539559e3dd0e869d9985f41262c30058d1d3c2cd336649b3a893bd4b29c206c53eeb337025a7585fc4bc05d8fe71035ea082ea543cbc15f8e6f0658815c7553795917958b503779cf6a18f92c391769e846327d3cd8458c089cb1e342a590eb57a583365c4bd5de34dc17992f5309fb69b67aa8436c763183a8f11a543fdb8a376a30836013fff3a2d21b72e22f1d9d031a3cd467365256e120894e1489c238df5ce65183b93d68024697245be8c0a9d60e31452d247a98136ca559622e2f4b35569ec8ab8539528a09214f1d3862bc97b21b7bb2b66c654ebee00eb26ec57988ff174cab65d4864eea14f15b65252fe534750bea3b9b4400c811ccf3e6517ecf1f1585d908f21b91bfd223eddbb9b980bb65133934c8027fde78865ed4d969bc6de3be613730d3a0f179f0ed734c67900da53c230045d60d31980401e7fa1396891d42555af8152bb0e6557d3c7d718f7ba85b6848ab052fcf709c7aea676d8c00c787a7fb2a2b1972d245aa51a24ceb13d7dd38cd0c214727a1000561828a62c969afa60d36cfab21b8e6837a0e0ac325c6c20e3ccd09fd1f5400dd6a7dc861e050bcb47ee89670d807b514e5b59ea830ef885ac6a0371efdcbb09e676fd9790e00ab9686f59b2ce173896d4d451e6fd61f3b95b82e63a494d7853c78c2f03e45cbabd46a4586364c131000d9efdc71d30d3caf25cdfad63a6628e67d5f219a0cafc5509f556cec7ac110ce0b52b552be6780e8d0c31a068dc6607ebc9cac4471cb1e85e5b6a0bc2330062f6930e2ef623da059da02902fa28586614a053625ea5901dc6151e7fae3a63b444b9d53630dea6b90d3c2ca7dd8db69f39a2ad6bc2eec08a85c1c0d0a9b079825278cf0b6f415344b0b6e4c7a51947e98ada9149c8f427914d8245b152f8f3558178d16e35498649a34ceb2fdaafc0d303829ddd9412c9a2b5ed1ce060472a9a85ffa19a4ddfbdb89437e72b261472d2b8c70a01a96562b753692c7329d75057a9918dbb3a39a90c86b66ec745a14e9909b2eae6c46dcd8c8a666973ba356124d6444353f1a34f88f907af897ea8e642f8603aed400d3d8a75568baa96b26b8d04f5fb0d1ca1256e7057f93464d95d1994ac5189ca607c013637fa35879a6c67c4806b6c9f00ccedb953103bbec21f054d304b621e0eca771accf5181409642d58ea8d032e06c27295e57092b4e3a89f47f47f16dc82f07dfad44ca7077385f62b4ad2efecb97955759c31977719316545b6fc6a78b7e35206719fcba4c1dca5d0f7f9959f7bd1c117532a2f7fc3ca87820e38cfab558dc48adb1058964b6ea9dfde5e06bff6b5d7af19fe6b8a46d0ec76f8902e525082591b82c2f86a30341787a1da86ab517063b39ae076b3f10e78f6c79dd58041b99ab40824b598f6c815733d62d262abeb96b8224b56e4109d4d5c445cd055a9cd2aea7753fc0f6a55bcf4291551a0d9c852e19e0f0603eefc9a79a38bc07a73f6e0f88a9d3eac01ca57d5d55e2ef870743522c15841b4ea7a0d278922cc977724571c75b5039a20f5dad843941d91fb421944b54923ed8c5f71edafbdfaf4c12dbe1c2b2a9642bbc2bd07017a44cb402d6ae09248eb24b70d43f751d3bf761ecef51f1f15239de25222852f95607335b3410050e5bd65f8c9657f0a95185e48dcb2f711641c0994e352b0039d569ddbd27182c263649d3e27b3816acd5822b31c98661adf01ee62db361be1b0469de4fc1c6e10185203eb44abe3d077fcbc52bef6095ca30c81a22b6e7a64c1385e7dd11209fc073a29ef656769fbb24d4292d98379884a1a2b2c7f6fd895577d739a2f0411e8afcee4de40927443f28e72aebc763fd0315ee85d29a23bb235ce82a5a621c0d9021965673720be9057a26b1ff0380263b54777f6cc7b7531d284f5041d4480108e4b4986f1ddec9e65dd2392c7fa0349c2a39954e4aeec8fe8f40e66db57d29beaf0046d9387482bab71a9397d90611cf767637c8666da87d5f1d798558cbb7228844010510cb95b073cec3878893ba70549eb6d3428b8db6944118f6de2ee7107b593ef85441cba46238f7843c4e6e7497f14cc64c25653c87da756226ce774c2b5e43294f2ebab2f601e9fe3f2d1d3172cfcc7e6eb7ea9b237e5f093443b02f42b4ea85673a6a0000ef9a6ddf2263a1a75eb7b78d3f90b1de91a22d78aa06d9626b0f9090ee63d92418b084db647132e3b0b7f3ed583eec280d06b94a358daffcd333233fb390ba8bf2da3921b18adf6cd4901cbabf4f4e3c90f21eb3190c8c0e4f16ac25dd546859bc640354ab9769553aeda466191ba4b10f52da5342347685e52af5d20ba8c113e65663bcedc12c99e576c4e1bdde013017d16fc26e3f30418c21d72ad6507ef1c8d437342d0fc20ad102e6c49eb9a8e7a3df5366c9a75b6d95ab007a3d93bca0086414ac5bc44a872659f43f0f703b415ac0e9aeeedb2c0cb945938923dc0865c5d3ff673e068d2865b11c68774cd6c0be1caa40627425bdc4ecbbd0a642f9c6953464e2f20681994be8483d64ed6d2d8efa79c5b12776900e58b45bea18c2e0220ea27e9670485e2c6ce9f52ac08cad61ca57b839710209db8a5d4fe95a846960b126271e519545e5d4d15300fc0ef2f35b8def7ba6f639d85404b57bac35140bef1c4f3b455773bf2a2eae62118dcdc5474ec0900aca49300833417786d1fb0d76a56570cfa10d23dbab0305e1c9c48032b19fd2ec2e00b1528a248036590e26e8c75209d004e20bc7730b29cf3ed860848b83ab91ad6a635f2cc1eca89e16814f34f2c1c2766a28b2901170bb4839f08f5685e593b8a5ce2a801194818b4aeb0a01794f92c7bc4144808e997cfd1711485b60483cf4a310b23e0210c5c73e6956b9e1ae696f1ea79b3fab788bf229a349fd9caedebd5db99821b1cebdceafb011cfcd1c78f93774b35f25c7e69952b0bca1a0d40791b812614e996ca31548bc0000000018cfdaa0e0052000288781053299010a001220757d23cec40eef9728417e68a5c6fb70b35e4fe58d34da55f305b013d24795a91802226b483045022100ae926c96c746308e7488e022f4ad1db94d5d0c8683f6fa6ded3afb13d8e20578022074aa8ebfe20adaf25beed70c60cfb5007278bec17875f11f7a5b3c86eb60a96901210391abdcd113c40b56f13a548e8624d6ad8d7a162b33ef81020d046cadbda2663728feffffff0f3a4110001a1976a914d79c1c8b67a0275c60e33b67bbd0e19a79b9276388ac222251477652524658424d32666558755533394577484e75595953726e4d33486155794d3a2910011a256a23426c6f636b73747265616d2e696e666f206973206120636f6f6c206578706c6f7265723a060a02048010024002" +) + +func init() { + testTx1 = bchain.Tx{ + Hex: "020000000101a99547d213b005f355da348de54f5eb370fbc6a5687e412897ef0ec4ce237d75020000006b483045022100ae926c96c746308e7488e022f4ad1db94d5d0c8683f6fa6ded3afb13d8e20578022074aa8ebfe20adaf25beed70c60cfb5007278bec17875f11f7a5b3c86eb60a96901210391abdcd113c40b56f13a548e8624d6ad8d7a162b33ef81020d046cadbda26637feffffff030af62b535fc393152f6d575708b271e3a53514cdcf65508c7d12e8fd06709ce24208e192e1315aa94c3bc0eddbb0ae195aff0a1ba2e19773b79e5d1b29fdd8df211b02f8b255b83fa13f40745fb5054b89dc7dcba497c850086a8726bac66ac22d4be31976a914d79c1c8b67a0275c60e33b67bbd0e19a79b9276388ac016d521c38ec1ea15734ae22b7c46064412829c0d0579f0a713d1c04ede979026f01000000000000000000256a23426c6f636b73747265616d2e696e666f206973206120636f6f6c206578706c6f726572016d521c38ec1ea15734ae22b7c46064412829c0d0579f0a713d1c04ede979026f0100000000000004800000000000000000000043010001db986007e38ccb7bdef1661fcf633cbde3d8850cd116736d9b0b0b027901921694954b26036eb51d9424c38083aa3d937998ea81077843925513523eb6ec3337fd4d0b602300000000000000011486003b49c8ab1a6aa09a8eaec4ed83dbda4cb4ef9651114513823a6eefb1dbf971631d88a229a3b027a94f9f15966765f6f5ce3245057c57aedf8f4c69e355f4264a10e2c7dad691879844074ae5e37152c828aba89035e928c63b1c1590638333d05ed08442538ebf7dff9c94ce11ab6bb6d5c9535fce99fab023aabc7215b52eed15454333428b065fb5d8b03aa2fe1de5004f9d8fca717bcd682f1f9caa6561bdafbe69c8423166f7e33867f0bc4fdd85224ff1533839762530d47a1f053c40c7e38f84a1431ad03398bc9d634384aec0f22e75ac4a94e3703ec8715d0791564a8b1509eab4bf7543ef2e5c20fe9fb6c2a85afe42fcdba64c628103ea5693287a28ded79f517a3e877fe287ca6f3d1229295abaf5c1b3b43ab54ec4697c941da47e0934718b697a414d8fd1a722eb23ceb554afb1b4c807a94507a35b153f19da10c6688c71efa0d1ad15c8ac2f3d786ae8c43136cacf333c2643e521322314346b201427f1ac975340cf3caf102245d8cf45709bb51e41fd357f1fb7316138f1992e5646842eb4fe18ef6a7096f7e4c1e69950285f30c64dc1da661c055944356f140d6298f5eaf733799bef034a8afb05f6f74239e572acf5d1c2f9e028b806caab13eb19148456b3db0f719ad76818eaea11e7989e74d858743ffba738a5f5c7d12ab1b049c594656821620e20817c6acc5b3d3725f9183d3c01a0190d5d991d1603682418f4a9c55b59cab8787f463123e1efd7d3e7f75dfb9ccf931ae2dc965b82cabf2af6b93293a4b00d7145d97a24679076157bade5ba4d7577922052c719ebf2493f5e9d0cf8e6f82666e8732dd91068395e528d0ea532e27ae8a84ed516dabd30dd990a1539d93f7b775039d53f1a12d7e7747a64f6aca9d7589330c8adf8854dce0fee0be212716d6c1b075fabb9d0c815e613c772f7c57e7efdb4e64a5ef47c9cf384deab55191d7b4b0cb27986649eca2c4286a01a7d3c304f008c6d3fe53abf320afc7602654cf4b69b87fd2b9b6794fe7cc77e92bbef572dc8ddb1530da6d82268cd32db112afb8c2db69651959c1ad39d804178cae05f856539559e3dd0e869d9985f41262c30058d1d3c2cd336649b3a893bd4b29c206c53eeb337025a7585fc4bc05d8fe71035ea082ea543cbc15f8e6f0658815c7553795917958b503779cf6a18f92c391769e846327d3cd8458c089cb1e342a590eb57a583365c4bd5de34dc17992f5309fb69b67aa8436c763183a8f11a543fdb8a376a30836013fff3a2d21b72e22f1d9d031a3cd467365256e120894e1489c238df5ce65183b93d68024697245be8c0a9d60e31452d247a98136ca559622e2f4b35569ec8ab8539528a09214f1d3862bc97b21b7bb2b66c654ebee00eb26ec57988ff174cab65d4864eea14f15b65252fe534750bea3b9b4400c811ccf3e6517ecf1f1585d908f21b91bfd223eddbb9b980bb65133934c8027fde78865ed4d969bc6de3be613730d3a0f179f0ed734c67900da53c230045d60d31980401e7fa1396891d42555af8152bb0e6557d3c7d718f7ba85b6848ab052fcf709c7aea676d8c00c787a7fb2a2b1972d245aa51a24ceb13d7dd38cd0c214727a1000561828a62c969afa60d36cfab21b8e6837a0e0ac325c6c20e3ccd09fd1f5400dd6a7dc861e050bcb47ee89670d807b514e5b59ea830ef885ac6a0371efdcbb09e676fd9790e00ab9686f59b2ce173896d4d451e6fd61f3b95b82e63a494d7853c78c2f03e45cbabd46a4586364c131000d9efdc71d30d3caf25cdfad63a6628e67d5f219a0cafc5509f556cec7ac110ce0b52b552be6780e8d0c31a068dc6607ebc9cac4471cb1e85e5b6a0bc2330062f6930e2ef623da059da02902fa28586614a053625ea5901dc6151e7fae3a63b444b9d53630dea6b90d3c2ca7dd8db69f39a2ad6bc2eec08a85c1c0d0a9b079825278cf0b6f415344b0b6e4c7a51947e98ada9149c8f427914d8245b152f8f3558178d16e35498649a34ceb2fdaafc0d303829ddd9412c9a2b5ed1ce060472a9a85ffa19a4ddfbdb89437e72b261472d2b8c70a01a96562b753692c7329d75057a9918dbb3a39a90c86b66ec745a14e9909b2eae6c46dcd8c8a666973ba356124d6444353f1a34f88f907af897ea8e642f8603aed400d3d8a75568baa96b26b8d04f5fb0d1ca1256e7057f93464d95d1994ac5189ca607c013637fa35879a6c67c4806b6c9f00ccedb953103bbec21f054d304b621e0eca771accf5181409642d58ea8d032e06c27295e57092b4e3a89f47f47f16dc82f07dfad44ca7077385f62b4ad2efecb97955759c31977719316545b6fc6a78b7e35206719fcba4c1dca5d0f7f9959f7bd1c117532a2f7fc3ca87820e38cfab558dc48adb1058964b6ea9dfde5e06bff6b5d7af19fe6b8a46d0ec76f8902e525082591b82c2f86a30341787a1da86ab517063b39ae076b3f10e78f6c79dd58041b99ab40824b598f6c815733d62d262abeb96b8224b56e4109d4d5c445cd055a9cd2aea7753fc0f6a55bcf4291551a0d9c852e19e0f0603eefc9a79a38bc07a73f6e0f88a9d3eac01ca57d5d55e2ef870743522c15841b4ea7a0d278922cc977724571c75b5039a20f5dad843941d91fb421944b54923ed8c5f71edafbdfaf4c12dbe1c2b2a9642bbc2bd07017a44cb402d6ae09248eb24b70d43f751d3bf761ecef51f1f15239de25222852f95607335b3410050e5bd65f8c9657f0a95185e48dcb2f711641c0994e352b0039d569ddbd27182c263649d3e27b3816acd5822b31c98661adf01ee62db361be1b0469de4fc1c6e10185203eb44abe3d077fcbc52bef6095ca30c81a22b6e7a64c1385e7dd11209fc073a29ef656769fbb24d4292d98379884a1a2b2c7f6fd895577d739a2f0411e8afcee4de40927443f28e72aebc763fd0315ee85d29a23bb235ce82a5a621c0d9021965673720be9057a26b1ff0380263b54777f6cc7b7531d284f5041d4480108e4b4986f1ddec9e65dd2392c7fa0349c2a39954e4aeec8fe8f40e66db57d29beaf0046d9387482bab71a9397d90611cf767637c8666da87d5f1d798558cbb7228844010510cb95b073cec3878893ba70549eb6d3428b8db6944118f6de2ee7107b593ef85441cba46238f7843c4e6e7497f14cc64c25653c87da756226ce774c2b5e43294f2ebab2f601e9fe3f2d1d3172cfcc7e6eb7ea9b237e5f093443b02f42b4ea85673a6a0000ef9a6ddf2263a1a75eb7b78d3f90b1de91a22d78aa06d9626b0f9090ee63d92418b084db647132e3b0b7f3ed583eec280d06b94a358daffcd333233fb390ba8bf2da3921b18adf6cd4901cbabf4f4e3c90f21eb3190c8c0e4f16ac25dd546859bc640354ab9769553aeda466191ba4b10f52da5342347685e52af5d20ba8c113e65663bcedc12c99e576c4e1bdde013017d16fc26e3f30418c21d72ad6507ef1c8d437342d0fc20ad102e6c49eb9a8e7a3df5366c9a75b6d95ab007a3d93bca0086414ac5bc44a872659f43f0f703b415ac0e9aeeedb2c0cb945938923dc0865c5d3ff673e068d2865b11c68774cd6c0be1caa40627425bdc4ecbbd0a642f9c6953464e2f20681994be8483d64ed6d2d8efa79c5b12776900e58b45bea18c2e0220ea27e9670485e2c6ce9f52ac08cad61ca57b839710209db8a5d4fe95a846960b126271e519545e5d4d15300fc0ef2f35b8def7ba6f639d85404b57bac35140bef1c4f3b455773bf2a2eae62118dcdc5474ec0900aca49300833417786d1fb0d76a56570cfa10d23dbab0305e1c9c48032b19fd2ec2e00b1528a248036590e26e8c75209d004e20bc7730b29cf3ed860848b83ab91ad6a635f2cc1eca89e16814f34f2c1c2766a28b2901170bb4839f08f5685e593b8a5ce2a801194818b4aeb0a01794f92c7bc4144808e997cfd1711485b60483cf4a310b23e0210c5c73e6956b9e1ae696f1ea79b3fab788bf229a349fd9caedebd5db99821b1cebdceafb011cfcd1c78f93774b35f25c7e69952b0bca1a0d40791b812614e996ca31548bc00000000", + Blocktime: 1544039759, + Time: 1544039759, + Txid: "7aa1af9481f2d744c96015b1baea6ba753790971ff265adfd93775de0234bfd6", + LockTime: 0, + Version: 2, + Vin: []bchain.Vin{ + { + ScriptSig: bchain.ScriptSig{ + Hex: "483045022100ae926c96c746308e7488e022f4ad1db94d5d0c8683f6fa6ded3afb13d8e20578022074aa8ebfe20adaf25beed70c60cfb5007278bec17875f11f7a5b3c86eb60a96901210391abdcd113c40b56f13a548e8624d6ad8d7a162b33ef81020d046cadbda26637", + }, + Txid: "757d23cec40eef9728417e68a5c6fb70b35e4fe58d34da55f305b013d24795a9", + Vout: 2, + Sequence: 4294967294, + }, + }, + Vout: []bchain.Vout{ + { + N: 0, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "76a914d79c1c8b67a0275c60e33b67bbd0e19a79b9276388ac", + Addresses: []string{ + "QGvRRFXBM2feXuU39EwHNuYYSrnM3HaUyM", + }, + }, + }, + { + ValueSat: *big.NewInt(0), + N: 1, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "6a23426c6f636b73747265616d2e696e666f206973206120636f6f6c206578706c6f726572", + }, + }, + { + ValueSat: *big.NewInt(1152), + N: 2, + ScriptPubKey: bchain.ScriptPubKey{ + Hex: "", + }, + }, + }, + } +} + +func Test_PackTx(t *testing.T) { + type args struct { + tx bchain.Tx + height uint32 + blockTime int64 + parser *LiquidParser + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "liquid-1", + args: args{ + tx: testTx1, + height: 82055, + blockTime: 1544039759, + parser: NewLiquidParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: testTxPacked1, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.args.parser.PackTx(&tt.args.tx, tt.args.height, tt.args.blockTime) + if (err != nil) != tt.wantErr { + t.Errorf("packTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + h := hex.EncodeToString(got) + if !reflect.DeepEqual(h, tt.want) { + t.Errorf("packTx() = %v, want %v", h, tt.want) + } + }) + } +} + +func Test_UnpackTx(t *testing.T) { + type args struct { + packedTx string + parser *LiquidParser + } + tests := []struct { + name string + args args + want *bchain.Tx + want1 uint32 + wantErr bool + }{ + { + name: "liquid-1", + args: args{ + packedTx: testTxPacked1, + parser: NewLiquidParser(GetChainParams("main"), &btc.Configuration{}), + }, + want: &testTx1, + want1: 82055, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, _ := hex.DecodeString(tt.args.packedTx) + got, got1, err := tt.args.parser.UnpackTx(b) + if (err != nil) != tt.wantErr { + t.Errorf("unpackTx() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unpackTx() got = %+v, want %+v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("unpackTx() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/bchain/coins/liquid/liquidrpc.go b/bchain/coins/liquid/liquidrpc.go new file mode 100644 index 00000000..7bfc54a7 --- /dev/null +++ b/bchain/coins/liquid/liquidrpc.go @@ -0,0 +1,108 @@ +package liquid + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/btc" + "encoding/json" + + "github.com/golang/glog" + "github.com/juju/errors" +) + +// LiquidRPC is an interface to JSON-RPC bitcoind service. +type LiquidRPC struct { + *btc.BitcoinRPC +} + +// NewLiquidRPC returns new LiquidRPC instance. +func NewLiquidRPC(config json.RawMessage, pushHandler func(bchain.NotificationType)) (bchain.BlockChain, error) { + b, err := btc.NewBitcoinRPC(config, pushHandler) + if err != nil { + return nil, err + } + + s := &LiquidRPC{ + b.(*btc.BitcoinRPC), + } + s.RPCMarshaler = btc.JSONMarshalerV2{} + + return s, nil +} + +// Initialize initializes GameCreditsRPC instance. +func (b *LiquidRPC) Initialize() error { + chainName, err := b.GetChainInfoAndInitializeMempool(b) + if err != nil { + return err + } + + glog.Info("Chain name ", chainName) + params := GetChainParams(chainName) + + // always create parser + b.Parser = NewLiquidParser(params, b.ChainConfig) + + // parameters for getInfo request + if params.Net == MainnetMagic { + b.Testnet = false + b.Network = "livenet" + } else { + b.Testnet = true + b.Network = "testnet" + } + + glog.Info("rpc: block chain ", params.Name) + + return nil +} + +// GetBlock returns block with given hash. +func (b *LiquidRPC) GetBlock(hash string, height uint32) (*bchain.Block, error) { + var err error + if hash == "" { + hash, err = b.GetBlockHash(height) + if err != nil { + return nil, err + } + } + + glog.V(1).Info("rpc: getblock (verbosity=1) ", hash) + + res := btc.ResGetBlockThin{} + req := btc.CmdGetBlock{Method: "getblock"} + req.Params.BlockHash = hash + req.Params.Verbosity = 1 + err = b.Call(&req, &res) + + if err != nil { + return nil, errors.Annotatef(err, "hash %v", hash) + } + if res.Error != nil { + return nil, errors.Annotatef(res.Error, "hash %v", hash) + } + + txs := make([]bchain.Tx, 0, len(res.Result.Txids)) + for _, txid := range res.Result.Txids { + tx, err := b.GetTransaction(txid) + if err != nil { + return nil, err + } + txs = append(txs, *tx) + } + block := &bchain.Block{ + BlockHeader: res.Result.BlockHeader, + Txs: txs, + } + return block, nil +} + +// GetTransactionForMempool returns a transaction by the transaction ID. +// It could be optimized for mempool, i.e. without block time and confirmations +func (b *LiquidRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) { + return b.GetTransaction(txid) +} + +// GetMempoolEntry returns mempool data for given transaction +func (b *LiquidRPC) GetMempoolEntry(txid string) (*bchain.MempoolEntry, error) { + return nil, errors.New("GetMempoolEntry: not implemented") +} diff --git a/bchain/coins/zec/zcashparser_test.go b/bchain/coins/zec/zcashparser_test.go index 18a6b4a7..65aa4afb 100644 --- a/bchain/coins/zec/zcashparser_test.go +++ b/bchain/coins/zec/zcashparser_test.go @@ -18,8 +18,8 @@ import ( var ( testTx1, testTx2 bchain.Tx - testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a490a05043c1aec8e10001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e76317951624441" - testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a490a050184a7a72310001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a470a0398968010011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a6970646467" + testTxPacked1 = "0a20e64aac0c211ad210c90934f06b1cc932327329e41a9f70c6eb76f79ef798b7b812ab1002000000019c012650c99d0ef761e863dbb966babf2cb7a7a2b5d90b1461c09521c473d23d000000006b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c83ffffffff018eec1a3c040000001976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac000000000162b4fc6b0000000000000000000000006ffa88c89b74f0f82e24744296845a0d0113b132ff5dfc2af34e6418eb15206af53078c4dd475cf143cd9a427983f5993622464b53e3a37d2519a946492c3977e30f0866550b9097222993a439a39260ac5e7d36aef38c7fdd1df3035a2d5817a9c20526e38f52f822d4db9d2f0156c4119d786d6e3a060ca871df7fae9a5c3a9c921b38ddc6414b13d16aa807389c68016e54bd6a9eb3b23a6bc7bf152e6dba15e9ec36f95dab15ad8f4a92a9d0309bbd930ef24bb7247bf534065c1e2f5b42e2c80eb59f48b4da6ec522319e065f8c4e463f95cc7fcad8d7ee91608e3c0ffcaa44129ba2d2da45d9a413919eca41af29faaf806a3eeb823e5a6c51afb1ec709505d812c0306bd76061a0a62d207355ad44d1ffce2b9e1dfd0818f79bd0f8e4031116b71fee2488484f17818b80532865773166cd389929e8409bb94e3948bd2e0215ef96d4e29d094590fda0de50715c11ff47c03380bb1d31b14e5b4ad8a372ca0b03364ef85f086b8a8eb5c56c3b1aee33e2cfbf1b2be1a3fb41b14b2c432b5d04d54c058fa87a96ae1d65d61b79360d09acc1e25a883fd7ae9a2a734a03362903021401c243173e1050b5cdb459b9ffc07c95e920f026618952d3a800b2e47e03b902084aed7ee8466a65d34abdbbd292781564dcd9b7440029d48c2640ebc196d4b40217f2872c1d0c1c9c2abf1147d6a5a9501895bc92960bfa182ceeb76a658224f1022bc53c4c1cd6888d72a152dc1aec5ba8a1d750fb7e498bee844d3481e4b4cd210227f94f775744185c9f24571b7df0c1c694cb2d3e4e9b955ed0b1caad2b02b5702139c4fbba03f0e422b2f3e4fc822b4f58baf32e7cd217cdbdec8540cb13d6496f271959b72a05e130eeffbe5b9a7fcd2793347cd9c0ea695265669844c363190f690c52a600cf413c3f00bdc5e9d1539e0cc63f4ec2945e0d86e6304a6deb5651e73eac21add5a641dfc95ab56200ed40d81f76755aee4659334c17ed3841ca5a5ab22f923956be1d264be2b485a0de55404510ece5c73d6626798be688f9dc18b69846acfe897a357cc4afe31f57fea32896717f124290e68f36f849fa6ecf76e02087f8c19dbc566135d7fa2daca2d843b9cc5bc3897d35f1de7d174f6407658f4a3706c12cea53d880b4d8c4d45b3f0d210214f815be49a664021a4a44b4a63e06a41d76b46f9aa6bad248e8d1a974ae7bbae5ea8ac269447db91637a19346729083cad5aebd5ff43ea13d04783068e9136da321b1152c666d2995d0ca06b26541deac62f4ef91f0e4af445b18a5c2a17c96eada0b27f85bb26dfb8f16515114c6b9f88037e2b85b3b84b65822eb99c992d99d12dcf9c71e5b46a586016faf5758483a716566db95b42187c101df68ca0554824e1c23cf0302bea03ad0a146af57e91794a268b8c82d78211718c8b5fea286f5de72fc7dfffecddcc02413525c472cb26022641d4bec2b8b7e71a7beb9ee18b82632799498eeee9a351cb9431a8d1906d5164acdf351bd538c3e9d1da8a211fe1cd18c44e72d8cdf16ce3fc9551552c05d52846ea7ef619232102588395cc2bcce509a4e7f150262a76c15475496c923dfce6bfc05871467ee7c213b39ea365c010083e0b1ba8926d3a9e586d8b11c9bab2a47d888bc7cb1a226c0086a1530e295d0047547006f4c8f1c24cdd8e16bb3845749895dec95f03fcda97d3224f6875b1b7b1c819d2fd35dd30968a3c82bc480d10082caf9d9dda8f9ec649c136c7fa07978099d97eaf4abfdc9854c266979d3cfc868f60689b6e3098b6c52a21796fe7c259d9a0dadf1b6efa59297d4c8c902febe7acf826eed30d40d2ac5119be91b51f4839d94599872c9a93c3e2691294914034001d3a278cb4a84d4ae048c0201a97e4cf1341ee663a162f5b586355018b9e5e30624ccdbeacf7d0382afacaf45f08e84d30c50bcd4e55c3138377261deb4e8c2931cd3c51cee94a048ae4839517b6e6537a5c0148d3830a33fea719ef9b4fa437e4d5fecdb646397c19ee56a0973c362a81803895cdc67246352dc566689cb203f9ebda900a5537bbb75aa25ddf3d4ab87b88737a58d760e1d271f08265daae1fe056e71971a8b826e5b215a05b71f99315b167dd2ec78874189657acafac2b5eeb9a901913f55f7ab69e1f9b203504448d414e71098b932a2309db57257eb3fef9de2f2a5a69aa46747d7b827df838345d38b95772bdab8c178c45777b92e8773864964b8e12ae29dbc1b21bf6527589f6bec71ff1cbb9928477409811c2e8150c79c3f21027ee954863b716875d3e9adfc6fdb18cd57a49bb395ca5c42da56f3beb78aad3a7a487de34a870bca61f3cdec422061328c83c910ab32ea7403c354915b7ebee29e1fea5a75158197e4a68e103f017fd7de5a70148ee7ce59356b1a74f83492e14faaa6cd4870bcc004e6eb0114d3429b74ea98fe2851b4553467a7660074e69b040aa31220d0e405d9166dbaf15e3ae2d8ec3b049ed99d17e0743bb6a1a7c3890bbdb7117f7374ad7a59aa1ab47d10445b28f4bc033794a71f88a8bf024189e9d27f9dc5859a4296437585b215656f807aca9dad35747494a43b8a1cf38be2b18a13de32a262ab29f9ba271c4fbce1a470a8243ebf9e7fd37b09262314afbb9a7e180218a0f1c9d505200028b0eb113299010a0012203dd273c42195c061140bd9b5a2a7b72cbfba66b9db63e861f70e9dc95026019c1800226b483045022100f220f48c5267ef92a1e7a4d3b44fe9d97cce76eeba2785d45a0e2620b70e8d7302205640bc39e197ce19d95a98a3239af0f208ca289c067f80c97d8e411e61da5dee0121021721e83315fb5282f1d9d2a11892322df589bccd9cef45517b5fb3cfd3055c8328ffffffff0f3a490a05043c1aec8e10001a1976a9149bb8229741305d8316ba3ca6a8d20740ce33c24188ac222374315934794c31344143486141626a656d6b647057376e594e48576e763179516244414000" + testTxPacked2 = "0a20bb47a9dd926de63e9d4f8dac58c3f63f4a079569ed3b80e932274a80f60e58b512e20101000000019cafb5c287980e6e5afb47339f6c1c81136d8255f5bd5226b36b01288494c46f000000006b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea22880feffffff0223a7a784010000001976a914826f87806ddd4643730be99b41c98acc379e83db88ac80969800000000001976a914e395634b7684289285926d4c64db395b783720ec88ac6e75040018e4b1c9d50520eeea1128f9ea113299010a0012206fc4948428016bb32652bdf555826d13811c6c9f3347fb5a6e0e9887c2b5af9c1800226b483045022100c92b2f3c54918fa26288530c63a58197ea4974e5b6d92db792dd9717e6d9183c02204e577254213675466a6adad3ae6e9384cf8269fb2dd9943b86fac0c0ad8e3f98012102c99dab469e63b232488b3e7acb9cfcab7e5755f61aad318d9e06b38e5ea2288028feffffff0f3a490a050184a7a72310001a1976a914826f87806ddd4643730be99b41c98acc379e83db88ac22237431566d4854547770457477766f6a786f644e32435351714c596931687a59336341713a470a0398968010011a1976a914e395634b7684289285926d4c64db395b783720ec88ac222374316563784d587070685554525158474c586e56684a367563714433445a69706464674000" ) func init() { diff --git a/bchain/mempool_utxo.go b/bchain/mempool_bitcoin_type.go similarity index 72% rename from bchain/mempool_utxo.go rename to bchain/mempool_bitcoin_type.go index 86fe6a5e..db12ffd4 100644 --- a/bchain/mempool_utxo.go +++ b/bchain/mempool_bitcoin_type.go @@ -7,44 +7,38 @@ import ( "github.com/golang/glog" ) -// addrIndex and outpoint are used also in non utxo mempool type addrIndex struct { addrDesc string n int32 } -type outpoint struct { - txid string - vout int32 -} - type txidio struct { txid string io []addrIndex } -// UTXOMempool is mempool handle. -type UTXOMempool struct { +// MempoolBitcoinType is mempool handle. +type MempoolBitcoinType struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrDescToTx map[string][]outpoint + addrDescToTx map[string][]Outpoint chanTxid chan string chanAddrIndex chan txidio onNewTxAddr OnNewTxAddrFunc } -// NewUTXOMempool creates new mempool handler. +// NewMempoolBitcoinType creates new mempool handler. // For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process -func NewUTXOMempool(chain BlockChain, workers int, subworkers int) *UTXOMempool { - m := &UTXOMempool{ +func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int) *MempoolBitcoinType { + m := &MempoolBitcoinType{ chain: chain, chanTxid: make(chan string, 1), chanAddrIndex: make(chan txidio, 1), } for i := 0; i < workers; i++ { go func(i int) { - chanInput := make(chan outpoint, 1) + chanInput := make(chan Outpoint, 1) chanResult := make(chan *addrIndex, 1) for j := 0; j < subworkers; j++ { go func(j int) { @@ -68,7 +62,7 @@ func NewUTXOMempool(chain BlockChain, workers int, subworkers int) *UTXOMempool } // GetTransactions returns slice of mempool transactions for given address -func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { +func (m *MempoolBitcoinType) GetTransactions(address string) ([]Outpoint, error) { parser := m.chain.GetChainParser() addrDesc, err := parser.GetAddrDescFromAddress(address) if err != nil { @@ -78,44 +72,39 @@ func (m *UTXOMempool) GetTransactions(address string) ([]string, error) { } // GetAddrDescTransactions returns slice of mempool transactions for given address descriptor -func (m *UTXOMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]string, error) { +func (m *MempoolBitcoinType) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) { m.mux.Lock() defer m.mux.Unlock() - outpoints := m.addrDescToTx[string(addrDesc)] - txs := make([]string, 0, len(outpoints)) - for _, o := range outpoints { - txs = append(txs, o.txid) - } - return txs, nil + return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil } -func (m *UTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]outpoint) { +func (m *MempoolBitcoinType) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]Outpoint) { m.mux.Lock() defer m.mux.Unlock() m.txToInputOutput = newTxToInputOutput m.addrDescToTx = newAddrDescToTx } -func (m *UTXOMempool) getInputAddress(input outpoint) *addrIndex { - itx, err := m.chain.GetTransactionForMempool(input.txid) +func (m *MempoolBitcoinType) getInputAddress(input Outpoint) *addrIndex { + itx, err := m.chain.GetTransactionForMempool(input.Txid) if err != nil { - glog.Error("cannot get transaction ", input.txid, ": ", err) + glog.Error("cannot get transaction ", input.Txid, ": ", err) return nil } - if int(input.vout) >= len(itx.Vout) { - glog.Error("Vout len in transaction ", input.txid, " ", len(itx.Vout), " input.Vout=", input.vout) + if int(input.Vout) >= len(itx.Vout) { + glog.Error("Vout len in transaction ", input.Txid, " ", len(itx.Vout), " input.Vout=", input.Vout) return nil } - addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&itx.Vout[input.vout]) + addrDesc, err := m.chain.GetChainParser().GetAddrDescFromVout(&itx.Vout[input.Vout]) if err != nil { - glog.Error("error in addrDesc in ", input.txid, " ", input.vout, ": ", err) + glog.Error("error in addrDesc in ", input.Txid, " ", input.Vout, ": ", err) return nil } - return &addrIndex{string(addrDesc), ^input.vout} + return &addrIndex{string(addrDesc), ^input.Vout} } -func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResult chan *addrIndex) ([]addrIndex, bool) { +func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan Outpoint, chanResult chan *addrIndex) ([]addrIndex, bool) { tx, err := m.chain.GetTransactionForMempool(txid) if err != nil { glog.Error("cannot get transaction ", txid, ": ", err) @@ -133,7 +122,7 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } if m.onNewTxAddr != nil { - m.onNewTxAddr(tx.Txid, addrDesc, true) + m.onNewTxAddr(tx, addrDesc) } } dispatched := 0 @@ -141,7 +130,7 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul if input.Coinbase != "" { continue } - o := outpoint{input.Txid, int32(input.Vout)} + o := Outpoint{input.Txid, int32(input.Vout)} loop: for { select { @@ -170,7 +159,7 @@ func (m *UTXOMempool) getTxAddrs(txid string, chanInput chan outpoint, chanResul // Resync gets mempool transactions and maps outputs to transactions. // Resync is not reentrant, it should be called from a single thread. // Read operations (GetTransactions) are safe. -func (m *UTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { +func (m *MempoolBitcoinType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { start := time.Now() glog.V(1).Info("mempool: resync") m.onNewTxAddr = onNewTxAddr @@ -181,13 +170,13 @@ func (m *UTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { glog.V(2).Info("mempool: resync ", len(txs), " txs") // allocate slightly larger capacity of the maps newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) - newAddrDescToTx := make(map[string][]outpoint, len(m.addrDescToTx)+5) + newAddrDescToTx := make(map[string][]Outpoint, len(m.addrDescToTx)+5) dispatched := 0 onNewData := func(txid string, io []addrIndex) { if len(io) > 0 { newTxToInputOutput[txid] = io for _, si := range io { - newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], outpoint{txid, si.n}) + newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], Outpoint{txid, si.n}) } } } diff --git a/bchain/mempool_nonutxo.go b/bchain/mempool_ethereum_type.go similarity index 54% rename from bchain/mempool_nonutxo.go rename to bchain/mempool_ethereum_type.go index c0151c50..769d9812 100644 --- a/bchain/mempool_nonutxo.go +++ b/bchain/mempool_ethereum_type.go @@ -7,21 +7,21 @@ import ( "github.com/golang/glog" ) -// NonUTXOMempool is mempool handle of non UTXO chains -type NonUTXOMempool struct { +// MempoolEthereumType is mempool handle of EthereumType chains +type MempoolEthereumType struct { chain BlockChain mux sync.Mutex txToInputOutput map[string][]addrIndex - addrDescToTx map[string][]outpoint + addrDescToTx map[string][]Outpoint } -// NewNonUTXOMempool creates new mempool handler. -func NewNonUTXOMempool(chain BlockChain) *NonUTXOMempool { - return &NonUTXOMempool{chain: chain} +// NewMempoolEthereumType creates new mempool handler. +func NewMempoolEthereumType(chain BlockChain) *MempoolEthereumType { + return &MempoolEthereumType{chain: chain} } // GetTransactions returns slice of mempool transactions for given address -func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { +func (m *MempoolEthereumType) GetTransactions(address string) ([]Outpoint, error) { parser := m.chain.GetChainParser() addrDesc, err := parser.GetAddrDescFromAddress(address) if err != nil { @@ -31,28 +31,35 @@ func (m *NonUTXOMempool) GetTransactions(address string) ([]string, error) { } // GetAddrDescTransactions returns slice of mempool transactions for given address descriptor -func (m *NonUTXOMempool) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]string, error) { +func (m *MempoolEthereumType) GetAddrDescTransactions(addrDesc AddressDescriptor) ([]Outpoint, error) { m.mux.Lock() defer m.mux.Unlock() - outpoints := m.addrDescToTx[string(addrDesc)] - txs := make([]string, 0, len(outpoints)) - for _, o := range outpoints { - txs = append(txs, o.txid) - } - return txs, nil + return append([]Outpoint(nil), m.addrDescToTx[string(addrDesc)]...), nil } -func (m *NonUTXOMempool) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]outpoint) { +func (m *MempoolEthereumType) updateMappings(newTxToInputOutput map[string][]addrIndex, newAddrDescToTx map[string][]Outpoint) { m.mux.Lock() defer m.mux.Unlock() m.txToInputOutput = newTxToInputOutput m.addrDescToTx = newAddrDescToTx } +func appendAddress(io []addrIndex, i int32, a string, parser BlockChainParser) []addrIndex { + if len(a) > 0 { + addrDesc, err := parser.GetAddrDescFromAddress(a) + if err != nil { + glog.Error("error in input addrDesc in ", a, ": ", err) + return io + } + io = append(io, addrIndex{string(addrDesc), i}) + } + return io +} + // Resync gets mempool transactions and maps outputs to transactions. // Resync is not reentrant, it should be called from a single thread. // Read operations (GetTransactions) are safe. -func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { +func (m *MempoolEthereumType) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { start := time.Now() glog.V(1).Info("Mempool: resync") txs, err := m.chain.GetMempool() @@ -62,13 +69,13 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { parser := m.chain.GetChainParser() // allocate slightly larger capacity of the maps newTxToInputOutput := make(map[string][]addrIndex, len(m.txToInputOutput)+5) - newAddrDescToTx := make(map[string][]outpoint, len(m.addrDescToTx)+5) + newAddrDescToTx := make(map[string][]Outpoint, len(m.addrDescToTx)+5) for _, txid := range txs { io, exists := m.txToInputOutput[txid] if !exists { tx, err := m.chain.GetTransactionForMempool(txid) if err != nil { - glog.Error("cannot get transaction ", txid, ": ", err) + glog.Warning("cannot get transaction ", txid, ": ", err) continue } io = make([]addrIndex, 0, len(tx.Vout)+len(tx.Vin)) @@ -83,29 +90,34 @@ func (m *NonUTXOMempool) Resync(onNewTxAddr OnNewTxAddrFunc) (int, error) { if len(addrDesc) > 0 { io = append(io, addrIndex{string(addrDesc), int32(output.N)}) } - if onNewTxAddr != nil { - onNewTxAddr(tx.Txid, addrDesc, true) - } } for _, input := range tx.Vin { for i, a := range input.Addresses { - if len(a) > 0 { - addrDesc, err := parser.GetAddrDescFromAddress(a) - if err != nil { - glog.Error("error in input addrDesc in ", txid, " ", a, ": ", err) - continue - } - io = append(io, addrIndex{string(addrDesc), int32(^i)}) - if onNewTxAddr != nil { - onNewTxAddr(tx.Txid, addrDesc, false) - } + appendAddress(io, ^int32(i), a, parser) + } + } + t, err := parser.EthereumTypeGetErc20FromTx(tx) + if err != nil { + glog.Error("GetErc20FromTx for tx ", txid, ", ", err) + } else { + for i := range t { + io = appendAddress(io, ^int32(i+1), t[i].From, parser) + io = appendAddress(io, int32(i+1), t[i].To, parser) + } + } + if onNewTxAddr != nil { + sent := make(map[string]struct{}) + for _, si := range io { + if _, found := sent[si.addrDesc]; !found { + onNewTxAddr(tx, AddressDescriptor(si.addrDesc)) + sent[si.addrDesc] = struct{}{} } } } } newTxToInputOutput[txid] = io for _, si := range io { - newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], outpoint{txid, si.n}) + newAddrDescToTx[si.addrDesc] = append(newAddrDescToTx[si.addrDesc], Outpoint{txid, si.n}) } } m.updateMappings(newTxToInputOutput, newAddrDescToTx) diff --git a/bchain/tx.pb.go b/bchain/tx.pb.go index 2115993a..d2718069 100644 --- a/bchain/tx.pb.go +++ b/bchain/tx.pb.go @@ -35,6 +35,7 @@ type ProtoTransaction struct { Height uint32 `protobuf:"varint,5,opt,name=Height" json:"Height,omitempty"` Vin []*ProtoTransaction_VinType `protobuf:"bytes,6,rep,name=Vin" json:"Vin,omitempty"` Vout []*ProtoTransaction_VoutType `protobuf:"bytes,7,rep,name=Vout" json:"Vout,omitempty"` + Version int32 `protobuf:"varint,8,opt,name=Version" json:"Version,omitempty"` } func (m *ProtoTransaction) Reset() { *m = ProtoTransaction{} } @@ -91,6 +92,13 @@ func (m *ProtoTransaction) GetVout() []*ProtoTransaction_VoutType { return nil } +func (m *ProtoTransaction) GetVersion() int32 { + if m != nil { + return m.Version + } + return 0 +} + type ProtoTransaction_VinType struct { Coinbase string `protobuf:"bytes,1,opt,name=Coinbase" json:"Coinbase,omitempty"` Txid []byte `protobuf:"bytes,2,opt,name=Txid,proto3" json:"Txid,omitempty"` @@ -196,26 +204,27 @@ func init() { func init() { proto.RegisterFile("tx.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 331 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xc1, 0x6e, 0xea, 0x30, - 0x10, 0x94, 0x89, 0x5f, 0x80, 0x7d, 0xa0, 0x87, 0xf6, 0xf0, 0x14, 0xa1, 0x1e, 0x52, 0x4e, 0x39, - 0xe5, 0x40, 0xd5, 0x0f, 0x68, 0x7b, 0x41, 0x6a, 0x85, 0x90, 0x83, 0x72, 0x4f, 0x82, 0x05, 0x56, - 0xa9, 0x4d, 0x13, 0x47, 0x02, 0xa9, 0x3f, 0xd3, 0x73, 0x7f, 0xb2, 0xf2, 0x12, 0x42, 0x41, 0xea, - 0x6d, 0x67, 0xbc, 0xe3, 0x19, 0x4f, 0x02, 0x3d, 0xbb, 0x8f, 0x77, 0xa5, 0xb1, 0x06, 0xfd, 0xbc, - 0xd8, 0x64, 0x4a, 0x4f, 0x3e, 0x39, 0x8c, 0x16, 0x8e, 0x59, 0x96, 0x99, 0xae, 0xb2, 0xc2, 0x2a, - 0xa3, 0x11, 0x81, 0x2f, 0xf7, 0x6a, 0x15, 0xb0, 0x90, 0x45, 0x03, 0x41, 0x33, 0x8e, 0xc0, 0x9b, - 0xc9, 0x7d, 0xd0, 0x21, 0xca, 0x8d, 0x78, 0x03, 0xfd, 0xc7, 0xad, 0x29, 0x5e, 0xad, 0x7a, 0x93, - 0x81, 0x17, 0xb2, 0x88, 0x8b, 0x33, 0x81, 0x63, 0xe8, 0xbd, 0x9c, 0x0e, 0x79, 0xc8, 0xa2, 0xa1, - 0x68, 0x31, 0xfe, 0x07, 0x7f, 0x26, 0xd5, 0x7a, 0x63, 0x83, 0x3f, 0x74, 0xd2, 0x20, 0x9c, 0x82, - 0x97, 0x2a, 0x1d, 0xf8, 0xa1, 0x17, 0xfd, 0x9d, 0x86, 0xf1, 0x31, 0x62, 0x7c, 0x1d, 0x2f, 0x4e, - 0x95, 0x5e, 0x1e, 0x76, 0x52, 0xb8, 0x65, 0xbc, 0x07, 0x9e, 0x9a, 0xda, 0x06, 0x5d, 0x12, 0xdd, - 0xfe, 0x2e, 0x32, 0xb5, 0x25, 0x15, 0xad, 0x8f, 0xbf, 0x18, 0x74, 0x9b, 0x7b, 0x5c, 0xd4, 0x27, - 0xa3, 0x74, 0x9e, 0x55, 0x92, 0x9e, 0xdc, 0x17, 0x2d, 0x6e, 0xab, 0xe8, 0xfc, 0xa8, 0x02, 0x1b, - 0x4b, 0x8f, 0xc2, 0xd3, 0x8c, 0x13, 0x18, 0x24, 0x45, 0xa9, 0x76, 0x36, 0x51, 0x6b, 0xd7, 0x13, - 0xa7, 0xfd, 0x0b, 0xce, 0xf9, 0x24, 0xf2, 0xbd, 0x96, 0xba, 0x90, 0xcd, 0xc3, 0x5b, 0xec, 0xca, - 0x7c, 0x58, 0xad, 0x4a, 0x59, 0x55, 0xb2, 0xa2, 0x02, 0xfa, 0xe2, 0x4c, 0x8c, 0x3f, 0xa0, 0x77, - 0xca, 0xef, 0x6e, 0x49, 0xb3, 0x6d, 0x2d, 0x93, 0xcc, 0x36, 0x1f, 0xa8, 0xc5, 0x38, 0x00, 0x36, - 0xa7, 0xa8, 0x43, 0xc1, 0xe6, 0x18, 0xc1, 0xbf, 0xa3, 0xff, 0xa2, 0xce, 0x9f, 0xe5, 0xc1, 0xc5, - 0xf2, 0x48, 0x70, 0x4d, 0x5f, 0xba, 0xf3, 0x2b, 0xf7, 0xdc, 0xa7, 0x5f, 0xe6, 0xee, 0x3b, 0x00, - 0x00, 0xff, 0xff, 0x95, 0x9e, 0x00, 0x7a, 0x3e, 0x02, 0x00, 0x00, + // 346 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0x41, 0x6e, 0xea, 0x30, + 0x14, 0x94, 0x89, 0x09, 0xe1, 0x7d, 0xd0, 0x47, 0x5e, 0x54, 0x16, 0xea, 0x22, 0x65, 0x95, 0x55, + 0x16, 0x54, 0x3d, 0x40, 0xdb, 0x0d, 0x52, 0x2b, 0x84, 0x1c, 0x94, 0x7d, 0x12, 0x2c, 0xb0, 0x4a, + 0x6d, 0x9a, 0x38, 0x12, 0x48, 0xbd, 0x51, 0x8f, 0xd0, 0xcb, 0x55, 0x7e, 0x84, 0x50, 0x90, 0xba, + 0xf3, 0x8c, 0xdf, 0x64, 0xe6, 0x4d, 0x0c, 0x81, 0xdd, 0xc7, 0xbb, 0xd2, 0x58, 0xc3, 0xfc, 0xbc, + 0xd8, 0x64, 0x4a, 0x4f, 0xbe, 0x29, 0x8c, 0x16, 0x8e, 0x59, 0x96, 0x99, 0xae, 0xb2, 0xc2, 0x2a, + 0xa3, 0x19, 0x03, 0xba, 0xdc, 0xab, 0x15, 0x27, 0x21, 0x89, 0x06, 0x02, 0xcf, 0x6c, 0x04, 0xde, + 0x4c, 0xee, 0x79, 0x07, 0x29, 0x77, 0x64, 0xb7, 0xd0, 0x7f, 0xda, 0x9a, 0xe2, 0xcd, 0xaa, 0x77, + 0xc9, 0xbd, 0x90, 0x44, 0x54, 0x9c, 0x09, 0x36, 0x86, 0xe0, 0xf5, 0x74, 0x49, 0x43, 0x12, 0x0d, + 0x45, 0x8b, 0xd9, 0x0d, 0xf8, 0x33, 0xa9, 0xd6, 0x1b, 0xcb, 0xbb, 0x78, 0xd3, 0x20, 0x36, 0x05, + 0x2f, 0x55, 0x9a, 0xfb, 0xa1, 0x17, 0xfd, 0x9b, 0x86, 0xf1, 0x31, 0x62, 0x7c, 0x1d, 0x2f, 0x4e, + 0x95, 0x5e, 0x1e, 0x76, 0x52, 0xb8, 0x61, 0xf6, 0x00, 0x34, 0x35, 0xb5, 0xe5, 0x3d, 0x14, 0xdd, + 0xfd, 0x2d, 0x32, 0xb5, 0x45, 0x15, 0x8e, 0x33, 0x0e, 0xbd, 0x54, 0x96, 0x95, 0x32, 0x9a, 0x07, + 0x21, 0x89, 0xba, 0xe2, 0x04, 0xc7, 0x5f, 0x04, 0x7a, 0x8d, 0x83, 0x5b, 0xe2, 0xd9, 0x28, 0x9d, + 0x67, 0x95, 0xc4, 0x32, 0xfa, 0xa2, 0xc5, 0x6d, 0x49, 0x9d, 0x5f, 0x25, 0xb1, 0x26, 0x8c, 0x87, + 0x6b, 0x1d, 0x9d, 0x26, 0x30, 0x48, 0x8a, 0x52, 0xed, 0x6c, 0xa2, 0xd6, 0xae, 0x41, 0x8a, 0xf3, + 0x17, 0x9c, 0xf3, 0x49, 0xe4, 0x47, 0x2d, 0x75, 0x21, 0x9b, 0x4a, 0x5a, 0xec, 0x6a, 0x7e, 0x5c, + 0xad, 0x4a, 0x59, 0x55, 0xb2, 0xc2, 0x6a, 0xfa, 0xe2, 0x4c, 0x8c, 0x3f, 0x21, 0x38, 0x6d, 0xe6, + 0xbe, 0x92, 0x66, 0xdb, 0x5a, 0x26, 0x99, 0x6d, 0x7e, 0x5d, 0x8b, 0xd9, 0x00, 0xc8, 0x1c, 0xa3, + 0x0e, 0x05, 0x99, 0xb3, 0x08, 0xfe, 0x1f, 0xfd, 0x17, 0x75, 0xfe, 0x22, 0x0f, 0x2e, 0x96, 0x87, + 0x82, 0x6b, 0xfa, 0xd2, 0x9d, 0x5e, 0xb9, 0xe7, 0x3e, 0x3e, 0xa6, 0xfb, 0x9f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xa1, 0x51, 0x2e, 0xba, 0x58, 0x02, 0x00, 0x00, } diff --git a/bchain/tx.proto b/bchain/tx.proto index 61e06aed..cd5c7bc5 100644 --- a/bchain/tx.proto +++ b/bchain/tx.proto @@ -23,4 +23,5 @@ syntax = "proto3"; uint32 Height = 5; repeated VinType Vin = 6; repeated VoutType Vout = 7; + int32 Version = 8; } \ No newline at end of file diff --git a/bchain/types.go b/bchain/types.go index 749adde7..293d4e58 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -9,6 +9,16 @@ import ( "math/big" ) +// ChainType is type of the blockchain +type ChainType int + +const ( + // ChainBitcoinType is blockchain derived from bitcoin + ChainBitcoinType = ChainType(iota) + // ChainEthereumType is blockchain derived from ethereum + ChainEthereumType +) + // errors with specific meaning returned by blockchain rpc var ( // ErrBlockNotFound is returned when block is not found @@ -23,11 +33,19 @@ var ( ErrTxidMissing = errors.New("Txid missing") ) +// Outpoint is txid together with output (or input) index +type Outpoint struct { + Txid string + Vout int32 +} + +// ScriptSig contains data about input script type ScriptSig struct { // Asm string `json:"asm"` Hex string `json:"hex"` } +// Vin contains data about tx output type Vin struct { Coinbase string `json:"coinbase"` Txid string `json:"txid"` @@ -37,6 +55,7 @@ type Vin struct { Addresses []string `json:"addresses"` } +// ScriptPubKey contains data about output script type ScriptPubKey struct { // Asm string `json:"asm"` Hex string `json:"hex,omitempty"` @@ -44,6 +63,7 @@ type ScriptPubKey struct { Addresses []string `json:"addresses"` } +// Vout contains data about tx output type Vout struct { ValueSat big.Int JsonValue json.Number `json:"value"` @@ -61,11 +81,13 @@ type Tx struct { Vin []Vin `json:"vin"` Vout []Vout `json:"vout"` // BlockHash string `json:"blockhash,omitempty"` - Confirmations uint32 `json:"confirmations,omitempty"` - Time int64 `json:"time,omitempty"` - Blocktime int64 `json:"blocktime,omitempty"` + Confirmations uint32 `json:"confirmations,omitempty"` + Time int64 `json:"time,omitempty"` + Blocktime int64 `json:"blocktime,omitempty"` + CoinSpecificData interface{} `json:"-"` } +// Block is block header and list of transactions type Block struct { BlockHeader Txs []Tx `json:"tx"` @@ -93,6 +115,7 @@ type BlockInfo struct { Txids []string `json:"tx,omitempty"` } +// MempoolEntry is used to get data about mempool entry type MempoolEntry struct { Size uint32 `json:"size"` FeeSat big.Int @@ -110,6 +133,7 @@ type MempoolEntry struct { Depends []string `json:"depends"` } +// ChainInfo is used to get information about blockchain type ChainInfo struct { Chain string `json:"chain"` Blocks int `json:"blocks"` @@ -124,6 +148,7 @@ type ChainInfo struct { Warnings string `json:"warnings"` } +// RPCError defines rpc error returned by backend type RPCError struct { Code int `json:"code"` Message string `json:"message"` @@ -140,11 +165,29 @@ func (ad AddressDescriptor) String() string { return "ad:" + hex.EncodeToString(ad) } +// EthereumType specific + +// Erc20Contract contains info about ERC20 contract +type Erc20Contract struct { + Contract string `json:"contract"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals int `json:"decimals"` +} + +// Erc20Transfer contains a single ERC20 token transfer +type Erc20Transfer struct { + Contract string + From string + To string + Tokens big.Int +} + // OnNewBlockFunc is used to send notification about a new block type OnNewBlockFunc func(hash string, height uint32) // OnNewTxAddrFunc is used to send notification about a new transaction/address -type OnNewTxAddrFunc func(txid string, desc AddressDescriptor, isOutput bool) +type OnNewTxAddrFunc func(tx *Tx, desc AddressDescriptor) // BlockChain defines common interface to block chain daemon type BlockChain interface { @@ -167,29 +210,34 @@ type BlockChain interface { GetMempool() ([]string, error) GetTransaction(txid string) (*Tx, error) GetTransactionForMempool(txid string) (*Tx, error) - GetTransactionSpecific(txid string) (json.RawMessage, error) + GetTransactionSpecific(tx *Tx) (json.RawMessage, error) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) EstimateFee(blocks int) (big.Int, error) SendRawTransaction(tx string) (string, error) // mempool ResyncMempool(onNewTxAddr OnNewTxAddrFunc) (int, error) - GetMempoolTransactions(address string) ([]string, error) - GetMempoolTransactionsForAddrDesc(addrDesc AddressDescriptor) ([]string, error) + GetMempoolTransactions(address string) ([]Outpoint, error) + GetMempoolTransactionsForAddrDesc(addrDesc AddressDescriptor) ([]Outpoint, error) GetMempoolEntry(txid string) (*MempoolEntry, error) // parser GetChainParser() BlockChainParser + // EthereumType specific + EthereumTypeGetBalance(addrDesc AddressDescriptor) (*big.Int, error) + EthereumTypeGetNonce(addrDesc AddressDescriptor) (uint64, error) + EthereumTypeEstimateGas(params map[string]interface{}) (uint64, error) + EthereumTypeGetErc20ContractInfo(contractDesc AddressDescriptor) (*Erc20Contract, error) + EthereumTypeGetErc20ContractBalance(addrDesc, contractDesc AddressDescriptor) (*big.Int, error) } // BlockChainParser defines common interface to parsing and conversions of block chain data type BlockChainParser interface { - // chain configuration description - // UTXO chains need "inputs" column in db, that map transactions to transactions that spend them - // non UTXO chains have mapping of address to input and output transactions directly in "outputs" column in db - IsUTXOChain() bool - // KeepBlockAddresses returns number of blocks which are to be kept in blockaddresses column - // and used in case of fork - // if 0 the blockaddresses column is not used at all (usually non UTXO chains) + // type of the blockchain + GetChainType() ChainType + // KeepBlockAddresses returns number of blocks which are to be kept in blockTxs column + // to be used for rollbacks KeepBlockAddresses() int + // AmountDecimals returns number of decimal places in coin amounts + AmountDecimals() int // AmountToDecimalString converts amount in big.Int to string with decimal point in the correct place AmountToDecimalString(a *big.Int) string // AmountToBigInt converts amount in json.Number (string) to big.Int @@ -212,4 +260,6 @@ type BlockChainParser interface { PackBlockHash(hash string) ([]byte, error) UnpackBlockHash(buf []byte) (string, error) ParseBlock(b []byte) (*Block, error) + // EthereumType specific + EthereumTypeGetErc20FromTx(tx *Tx) ([]Erc20Transfer, error) } diff --git a/blockbook.go b/blockbook.go index 3ff1ac56..a939cf3a 100644 --- a/blockbook.go +++ b/blockbook.go @@ -498,17 +498,17 @@ func storeInternalStateLoop() { glog.Info("storeInternalStateLoop stopped") } -func onNewTxAddr(txid string, desc bchain.AddressDescriptor, isOutput bool) { +func onNewTxAddr(tx *bchain.Tx, desc bchain.AddressDescriptor) { for _, c := range callbacksOnNewTxAddr { - c(txid, desc, isOutput) + c(tx, desc) } } func pushSynchronizationHandler(nt bchain.NotificationType) { + glog.V(1).Info("MQ: notification ", nt) if atomic.LoadInt32(&inShutdown) != 0 { return } - glog.V(1).Info("MQ: notification ", nt) if nt == bchain.NotificationNewBlock { chanSyncIndex <- struct{}{} } else if nt == bchain.NotificationNewTx { @@ -545,7 +545,7 @@ func waitForSignalAndShutdown(internal *server.InternalServer, public *server.Pu } } -func printResult(txid string, vout uint32, isOutput bool) error { +func printResult(txid string, vout int32, isOutput bool) error { glog.Info(txid, vout, isOutput) return nil } diff --git a/common/internalstate.go b/common/internalstate.go index f19fd1f8..9419beaa 100644 --- a/common/internalstate.go +++ b/common/internalstate.go @@ -150,9 +150,7 @@ func (is *InternalState) GetDBColumnStatValues(c int) (int64, int64, int64) { func (is *InternalState) GetAllDBColumnStats() []InternalStateColumn { is.mux.Lock() defer is.mux.Unlock() - rv := make([]InternalStateColumn, len(is.DbColumns)) - copy(rv, is.DbColumns) - return rv + return append(is.DbColumns[:0:0], is.DbColumns...) } // DBSizeTotal sums the computed sizes of all columns diff --git a/common/metrics.go b/common/metrics.go index 6e338195..6f92b14b 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -11,6 +11,10 @@ type Metrics struct { SocketIOSubscribes *prometheus.CounterVec SocketIOClients prometheus.Gauge SocketIOReqDuration *prometheus.HistogramVec + WebsocketRequests *prometheus.CounterVec + WebsocketSubscribes *prometheus.CounterVec + WebsocketClients prometheus.Gauge + WebsocketReqDuration *prometheus.HistogramVec IndexResyncDuration prometheus.Histogram MempoolResyncDuration prometheus.Histogram TxCacheEfficiency *prometheus.CounterVec @@ -48,7 +52,7 @@ func GetMetrics(coin string) (*Metrics, error) { metrics.SocketIOClients = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "blockbook_socketio_clients", - Help: "Number of currently connected clients", + Help: "Number of currently connected socketio clients", ConstLabels: Labels{"coin": coin}, }, ) @@ -61,6 +65,38 @@ func GetMetrics(coin string) (*Metrics, error) { }, []string{"method"}, ) + metrics.WebsocketRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "blockbook_websocket_requests", + Help: "Total number of websocket requests by method and status", + ConstLabels: Labels{"coin": coin}, + }, + []string{"method", "status"}, + ) + metrics.WebsocketSubscribes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "blockbook_websocket_subscribes", + Help: "Total number of websocket subscribes by channel and status", + ConstLabels: Labels{"coin": coin}, + }, + []string{"channel", "status"}, + ) + metrics.WebsocketClients = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "blockbook_websocket_clients", + Help: "Number of currently connected websocket clients", + ConstLabels: Labels{"coin": coin}, + }, + ) + metrics.WebsocketReqDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "blockbook_websocket_req_duration", + Help: "Websocket request duration by method (in microseconds)", + Buckets: []float64{1, 5, 10, 25, 50, 75, 100, 250}, + ConstLabels: Labels{"coin": coin}, + }, + []string{"method"}, + ) metrics.IndexResyncDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Name: "blockbook_index_resync_duration", diff --git a/configs/coins/ethereum-classic.json b/configs/coins/ethereum-classic.json index e783c41b..93e4268f 100644 --- a/configs/coins/ethereum-classic.json +++ b/configs/coins/ethereum-classic.json @@ -40,7 +40,7 @@ "system_user": "blockbook-ethereum-classic", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://gastracker.io/", + "explorer_url": "", "additional_params": "-resyncindexperiod=4441", "block_chain": { "parse": true, diff --git a/configs/coins/ethereum.json b/configs/coins/ethereum.json index 33065e05..bd83b60b 100644 --- a/configs/coins/ethereum.json +++ b/configs/coins/ethereum.json @@ -42,7 +42,7 @@ "system_user": "blockbook-ethereum", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://etherscan.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/ethereum_testnet_ropsten.json b/configs/coins/ethereum_testnet_ropsten.json index 36e7348a..5cea42ff 100644 --- a/configs/coins/ethereum_testnet_ropsten.json +++ b/configs/coins/ethereum_testnet_ropsten.json @@ -1,8 +1,8 @@ { "coin": { "name": "Ethereum Testnet Ropsten", - "shortcut": "tETH", - "label": "Ethereum Testnet Ropsten", + "shortcut": "tROP", + "label": "Ethereum Ropsten", "alias": "ethereum_testnet_ropsten" }, "ports": { @@ -41,7 +41,7 @@ "system_user": "blockbook-ethereum", "internal_binding_template": ":{{.Ports.BlockbookInternal}}", "public_binding_template": ":{{.Ports.BlockbookPublic}}", - "explorer_url": "https://ropsten.etherscan.io/", + "explorer_url": "", "additional_params": "", "block_chain": { "parse": true, diff --git a/configs/coins/liquid.json b/configs/coins/liquid.json new file mode 100644 index 00000000..9ea54270 --- /dev/null +++ b/configs/coins/liquid.json @@ -0,0 +1,68 @@ +{ + "coin": { + "name": "Liquid", + "shortcut": "L-BTC", + "label": "Liquid", + "alias": "liquid" + }, + "ports": { + "backend_rpc": 8046, + "backend_message_queue": 38346, + "blockbook_internal": 9046, + "blockbook_public": 9146 + }, + "ipc": { + "rpc_url_template": "http://127.0.0.1:{{.Ports.BackendRPC}}", + "rpc_user": "rpc", + "rpc_pass": "rpc", + "rpc_timeout": 25, + "message_queue_binding_template": "tcp://127.0.0.1:{{.Ports.BackendMessageQueue}}" + }, + "backend": { + "package_name": "backend-liquid", + "package_revision": "satoshilabs-1", + "system_user": "liquid", + "version": "3.14.1.21", + "binary_url": "https://github.com/Blockstream/liquid/releases/download/liquid.3.14.1.21/liquid-3.14.1.21-x86_64-linux-gnu.tar.gz", + "verification_type": "sha256", + "verification_source": "ea2836aa267b32b29e890acdd5e724b4be225c34891fd26426ce741c12c1e166", + "extract_command": "tar -C backend --strip 1 -xf", + "exclude_files": [ + "bin/test_elements" + ], + "exec_command_template": "{{.Env.BackendInstallPath}}/{{.Coin.Alias}}/bin/liquidd -datadir={{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend -conf={{.Env.BackendInstallPath}}/{{.Coin.Alias}}/{{.Coin.Alias}}.conf -pid=/run/{{.Coin.Alias}}/{{.Coin.Alias}}.pid", + "logrotate_files_template": "{{.Env.BackendDataPath}}/{{.Coin.Alias}}/backend/liquidv1/*.log", + "postinst_script_template": "", + "service_type": "forking", + "service_additional_params_template": "", + "protect_memory": true, + "mainnet": true, + "server_config_file": "bitcoin_like.conf", + "client_config_file": "bitcoin_like_client.conf", + "additional_params": { + "deprecatedrpc": "estimatefee", + "mainchainrpcport": "8030", + "mainchainrpcuser": "rpc", + "mainchainrpcpassword": "rpc" + } + }, + "blockbook": { + "package_name": "blockbook-liquid", + "system_user": "blockbook-liquid", + "internal_binding_template": ":{{.Ports.BlockbookInternal}}", + "public_binding_template": ":{{.Ports.BlockbookPublic}}", + "explorer_url": "", + "additional_params": "", + "block_chain": { + "parse": true, + "mempool_workers": 8, + "mempool_sub_workers": 2, + "block_addresses_to_keep": 300, + "additional_params": {} + } + }, + "meta": { + "package_maintainer": "Martin Bohm", + "package_maintainer_email": "martin.bohm@satoshilabs.com" + } +} \ No newline at end of file diff --git a/db/bulkconnect.go b/db/bulkconnect.go index 60ca7a64..7221acad 100644 --- a/db/bulkconnect.go +++ b/db/bulkconnect.go @@ -22,7 +22,7 @@ type bulkAddresses struct { // BulkConnect is used to connect blocks in bulk, faster but if interrupted inconsistent way type BulkConnect struct { d *RocksDB - isUTXO bool + chainType bchain.ChainType bulkAddresses []bulkAddresses bulkAddressesCount int txAddressesMap map[string]*TxAddresses @@ -42,7 +42,7 @@ const ( func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) { bc := &BulkConnect{ d: d, - isUTXO: d.chainParser.IsUTXOChain(), + chainType: d.chainParser.GetChainType(), txAddressesMap: make(map[string]*TxAddresses), balances: make(map[string]*AddrBalance), } @@ -168,11 +168,12 @@ func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error { // ConnectBlock connects block in bulk mode func (b *BulkConnect) ConnectBlock(block *bchain.Block, storeBlockTxs bool) error { b.height = block.Height - if !b.isUTXO { + // for non bitcoin types connect blocks in non bulk mode + if b.chainType != bchain.ChainBitcoinType { return b.d.ConnectBlock(block) } addresses := make(map[string][]outpoint) - if err := b.d.processAddressesUTXO(block, addresses, b.txAddressesMap, b.balances); err != nil { + if err := b.d.processAddressesBitcoinType(block, addresses, b.txAddressesMap, b.balances); err != nil { return err } var storeAddressesChan, storeBalancesChan chan error diff --git a/db/rocksdb.go b/db/rocksdb.go index dcd8044c..555040f2 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -59,13 +59,21 @@ const ( cfDefault = iota cfHeight cfAddresses - cfTxAddresses - cfAddressBalance cfBlockTxs cfTransactions + // BitcoinType + cfAddressBalance + cfTxAddresses + // EthereumType + cfAddressContracts = cfAddressBalance ) -var cfNames = []string{"default", "height", "addresses", "txAddresses", "addressBalance", "blockTxs", "transactions"} +// common columns +var cfNames = []string{"default", "height", "addresses", "blockTxs", "transactions"} + +// type specific columns +var cfNamesBitcoinType = []string{"addressBalance", "txAddresses"} +var cfNamesEthereumType = []string{"addressContracts"} func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter @@ -73,9 +81,14 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g // opts for addresses without bloom filter // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter optsAddresses := createAndSetDBOptions(0, c, openFiles) - // default, height, addresses, txAddresses, addressBalance, blockTxids, transactions - fcOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts, opts} - db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, fcOptions) + // default, height, addresses, blockTxids, transactions + cfOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts} + // append type specific options + count := len(cfNames) - len(cfOptions) + for i := 0; i < count; i++ { + cfOptions = append(cfOptions, opts) + } + db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, cfOptions) if err != nil { return nil, nil, err } @@ -86,6 +99,16 @@ func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*g // needs to be called to release it. func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) + + chainType := parser.GetChainType() + if chainType == bchain.ChainBitcoinType { + cfNames = append(cfNames, cfNamesBitcoinType...) + } else if chainType == bchain.ChainEthereumType { + cfNames = append(cfNames, cfNamesEthereumType...) + } else { + return nil, errors.New("Unknown chain type") + } + c := gorocksdb.NewLRUCache(cacheSize) db, cfh, err := openDB(path, c, maxOpenFiles) if err != nil { @@ -185,7 +208,7 @@ func (e *StopIteration) Error() string { // GetTransactions finds all input/output transactions for address // Transaction are passed to callback function. -func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { +func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) { if glog.V(1) { glog.Infof("rocksdb: address get %s %d-%d ", address, lower, higher) } @@ -198,7 +221,7 @@ func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, f // GetAddrDescTransactions finds all input/output transactions for address descriptor // Transaction are passed to callback function. -func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout uint32, isOutput bool) error) (err error) { +func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn func(txid string, vout int32, isOutput bool) error) (err error) { kstart := packAddressKey(addrDesc, lower) kstop := packAddressKey(addrDesc, higher) @@ -219,13 +242,13 @@ func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, low glog.Infof("rocksdb: output %s: %s", hex.EncodeToString(key), hex.EncodeToString(val)) } for _, o := range outpoints { - var vout uint32 + var vout int32 var isOutput bool if o.index < 0 { - vout = uint32(^o.index) + vout = int32(^o.index) isOutput = false } else { - vout = uint32(o.index) + vout = int32(o.index) isOutput = true } tx, err := d.chainParser.UnpackTxid(o.btxID) @@ -250,45 +273,23 @@ const ( // ConnectBlock indexes addresses in the block and stores them in db func (d *RocksDB) ConnectBlock(block *bchain.Block) error { - return d.writeBlock(block, opInsert) -} - -// DisconnectBlock removes addresses in the block from the db -func (d *RocksDB) DisconnectBlock(block *bchain.Block) error { - return d.writeBlock(block, opDelete) -} - -func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { wb := gorocksdb.NewWriteBatch() defer wb.Destroy() if glog.V(2) { - switch op { - case opInsert: - glog.Infof("rocksdb: insert %d %s", block.Height, block.Hash) - case opDelete: - glog.Infof("rocksdb: delete %d %s", block.Height, block.Hash) - } + glog.Infof("rocksdb: insert %d %s", block.Height, block.Hash) } - isUTXO := d.chainParser.IsUTXOChain() + chainType := d.chainParser.GetChainType() - if err := d.writeHeightFromBlock(wb, block, op); err != nil { + if err := d.writeHeightFromBlock(wb, block, opInsert); err != nil { return err } - if isUTXO { - if op == opDelete { - // block does not contain mapping tx-> input address, which is necessary to recreate - // unspentTxs; therefore it is not possible to DisconnectBlocks this way - return errors.New("DisconnectBlock is not supported for UTXO chains") - } - addresses := make(map[string][]outpoint) + addresses := make(map[string][]outpoint) + if chainType == bchain.ChainBitcoinType { txAddressesMap := make(map[string]*TxAddresses) balances := make(map[string]*AddrBalance) - if err := d.processAddressesUTXO(block, addresses, txAddressesMap, balances); err != nil { - return err - } - if err := d.storeAddresses(wb, block.Height, addresses); err != nil { + if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances); err != nil { return err } if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { @@ -300,10 +301,23 @@ func (d *RocksDB) writeBlock(block *bchain.Block, op int) error { if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { return err } - } else { - if err := d.writeAddressesNonUTXO(wb, block, op); err != nil { + } else if chainType == bchain.ChainEthereumType { + addressContracts := make(map[string]*AddrContracts) + blockTxs, err := d.processAddressesEthereumType(block, addresses, addressContracts) + if err != nil { return err } + if err := d.storeAddressContracts(wb, addressContracts); err != nil { + return err + } + if err := d.storeAndCleanupBlockTxsEthereumType(wb, block, blockTxs); err != nil { + return err + } + } else { + return errors.New("Unknown chain type") + } + if err := d.storeAddresses(wb, block.Height, addresses); err != nil { + return err } return d.db.Write(d.wo, wb) @@ -375,7 +389,7 @@ func (d *RocksDB) GetAndResetConnectBlockStats() string { return s } -func (d *RocksDB) processAddressesUTXO(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { +func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses map[string][]outpoint, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { blockTxIDs := make([][]byte, len(block.Txs)) blockTxAddresses := make([]*TxAddresses, len(block.Txs)) // first process all outputs so that inputs can point to txs in this block @@ -577,6 +591,27 @@ func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBa return nil } +func (d *RocksDB) cleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { + keep := d.chainParser.KeepBlockAddresses() + // cleanup old block address + if block.Height > uint32(keep) { + for rh := block.Height - uint32(keep); rh > 0; rh-- { + key := packUint(rh) + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) + if err != nil { + return err + } + // nil data means the key was not found in DB + if val.Data() == nil { + break + } + val.Free() + d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) + } + } + return nil +} + func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { pl := d.chainParser.PackedTxidLen() buf := make([]byte, 0, pl*len(block.Txs)) @@ -610,23 +645,7 @@ func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchai } key := packUint(block.Height) wb.PutCF(d.cfh[cfBlockTxs], key, buf) - keep := d.chainParser.KeepBlockAddresses() - // cleanup old block address - if block.Height > uint32(keep) { - for rh := block.Height - uint32(keep); rh < block.Height; rh-- { - key = packUint(rh) - val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) - if err != nil { - return err - } - if val.Size() == 0 { - break - } - val.Free() - d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) - } - } - return nil + return d.cleanupBlockTxs(wb, block) } func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { @@ -643,8 +662,7 @@ func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) return nil, errors.New("Inconsistent data in blockTxs") } - txid := make([]byte, pl) - copy(txid, buf[i:]) + txid := append([]byte(nil), buf[i:i+pl]...) i += pl o, ol, err := d.unpackNOutpoints(buf[i:]) if err != nil { @@ -775,8 +793,7 @@ func unpackTxAddresses(buf []byte) (*TxAddresses, error) { func unpackTxInput(ti *TxInput, buf []byte) int { al, l := unpackVaruint(buf) - ti.AddrDesc = make([]byte, al) - copy(ti.AddrDesc, buf[l:l+int(al)]) + ti.AddrDesc = append([]byte(nil), buf[l:l+int(al)]...) al += uint(l) ti.ValueSat, l = unpackBigint(buf[al:]) return l + int(al) @@ -788,8 +805,7 @@ func unpackTxOutput(to *TxOutput, buf []byte) int { to.Spent = true al = ^al } - to.AddrDesc = make([]byte, al) - copy(to.AddrDesc, buf[l:l+al]) + to.AddrDesc = append([]byte(nil), buf[l:l+al]...) al += l to.ValueSat, l = unpackBigint(buf[al:]) return l + al @@ -842,74 +858,6 @@ func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { return outpoints, p, nil } -func (d *RocksDB) addAddrDescToRecords(op int, wb *gorocksdb.WriteBatch, records map[string][]outpoint, addrDesc bchain.AddressDescriptor, btxid []byte, vout int32, bh uint32) error { - if len(addrDesc) > 0 { - if len(addrDesc) > maxAddrDescLen { - glog.Infof("rocksdb: block %d, skipping addrDesc of length %d", bh, len(addrDesc)) - } else { - strAddrDesc := string(addrDesc) - records[strAddrDesc] = append(records[strAddrDesc], outpoint{ - btxID: btxid, - index: vout, - }) - if op == opDelete { - // remove transactions from cache - d.internalDeleteTx(wb, btxid) - } - } - } - return nil -} - -func (d *RocksDB) writeAddressesNonUTXO(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { - addresses := make(map[string][]outpoint) - for _, tx := range block.Txs { - btxID, err := d.chainParser.PackTxid(tx.Txid) - if err != nil { - return err - } - for _, output := range tx.Vout { - addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) - if err != nil { - // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) - if err != bchain.ErrAddressMissing { - glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v", err, block.Height, tx.Txid, output) - } - continue - } - err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(output.N), block.Height) - if err != nil { - return err - } - } - // store inputs in format txid ^index - for _, input := range tx.Vin { - for i, a := range input.Addresses { - addrDesc, err := d.chainParser.GetAddrDescFromAddress(a) - if err != nil { - glog.Warningf("rocksdb: addrDesc: %v - %d %s", err, block.Height, addrDesc) - continue - } - err = d.addAddrDescToRecords(op, wb, addresses, addrDesc, btxID, int32(^i), block.Height) - if err != nil { - return err - } - } - } - } - for addrDesc, outpoints := range addresses { - key := packAddressKey(bchain.AddressDescriptor(addrDesc), block.Height) - switch op { - case opInsert: - val := d.packOutpoints(outpoints) - wb.PutCF(d.cfh[cfAddresses], key, val) - case opDelete: - wb.DeleteCF(d.cfh[cfAddresses], key) - } - } - return nil -} - // Block index // BlockInfo holds information about blocks kept in column height @@ -1035,51 +983,6 @@ func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, height uint32, bi *Block // Disconnect blocks -func (d *RocksDB) allAddressesScan(lower uint32, higher uint32) ([][]byte, [][]byte, error) { - glog.Infof("db: doing full scan of addresses column") - addrKeys := [][]byte{} - addrValues := [][]byte{} - var totalOutputs, count uint64 - var seekKey []byte - for { - var key []byte - it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) - if totalOutputs == 0 { - it.SeekToFirst() - } else { - it.Seek(seekKey) - it.Next() - } - for count = 0; it.Valid() && count < refreshIterator; it.Next() { - totalOutputs++ - count++ - key = it.Key().Data() - l := len(key) - if l > packedHeightBytes { - height := unpackUint(key[l-packedHeightBytes : l]) - if height >= lower && height <= higher { - addrKey := make([]byte, len(key)) - copy(addrKey, key) - addrKeys = append(addrKeys, addrKey) - value := it.Value().Data() - addrValue := make([]byte, len(value)) - copy(addrValue, value) - addrValues = append(addrValues, addrValue) - } - } - } - seekKey = make([]byte, len(key)) - copy(seekKey, key) - valid := it.Valid() - it.Close() - if !valid { - break - } - } - glog.Infof("rocksdb: scanned %d addresses, found %d to disconnect", totalOutputs, len(addrKeys)) - return addrKeys, addrValues, nil -} - func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, txid string, inputs []outpoint, txa *TxAddresses, txAddressesToUpdate map[string]*TxAddresses, balances map[string]*AddrBalance) error { addresses := make(map[string]struct{}) @@ -1166,10 +1069,9 @@ func (d *RocksDB) disconnectTxAddresses(wb *gorocksdb.WriteBatch, height uint32, return nil } -// DisconnectBlockRangeUTXO removes all data belonging to blocks in range lower-higher -// if they are in the range kept in the cfBlockTxids column -func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) +// DisconnectBlockRangeBitcoinType removes all data belonging to blocks in range lower-higher +// it is able to disconnect only blocks for which there are data in the blockTxs column +func (d *RocksDB) DisconnectBlockRangeBitcoinType(lower uint32, higher uint32) error { blocks := make([][]blockTxs, higher-lower+1) for height := lower; height <= higher; height++ { blockTxs, err := d.getBlockTxs(height) @@ -1227,37 +1129,6 @@ func (d *RocksDB) DisconnectBlockRangeUTXO(lower uint32, higher uint32) error { return err } -// DisconnectBlockRangeNonUTXO performs full range scan to remove a range of blocks -// it is very slow operation -func (d *RocksDB) DisconnectBlockRangeNonUTXO(lower uint32, higher uint32) error { - glog.Infof("db: disconnecting blocks %d-%d", lower, higher) - addrKeys, _, err := d.allAddressesScan(lower, higher) - if err != nil { - return err - } - glog.Infof("rocksdb: about to disconnect %d addresses ", len(addrKeys)) - wb := gorocksdb.NewWriteBatch() - defer wb.Destroy() - for _, addrKey := range addrKeys { - if glog.V(2) { - glog.Info("address ", hex.EncodeToString(addrKey)) - } - // delete address:height from the index - wb.DeleteCF(d.cfh[cfAddresses], addrKey) - } - for height := lower; height <= higher; height++ { - if glog.V(2) { - glog.Info("height ", height) - } - wb.DeleteCF(d.cfh[cfHeight], packUint(height)) - } - err = d.db.Write(d.wo, wb) - if err == nil { - glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) - } - return err -} - func dirSize(path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { diff --git a/db/rocksdb_ethereumtype.go b/db/rocksdb_ethereumtype.go new file mode 100644 index 00000000..ad6a96e4 --- /dev/null +++ b/db/rocksdb_ethereumtype.go @@ -0,0 +1,409 @@ +package db + +import ( + "blockbook/bchain" + "blockbook/bchain/coins/eth" + "bytes" + "encoding/hex" + + "github.com/bsm/go-vlq" + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/tecbot/gorocksdb" +) + +// AddrContract is Contract address with number of transactions done by given address +type AddrContract struct { + Contract bchain.AddressDescriptor + Txs uint +} + +// AddrContracts is array of contracts with +type AddrContracts struct { + EthTxs uint + Contracts []AddrContract +} + +func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string]*AddrContracts) error { + buf := make([]byte, 64) + varBuf := make([]byte, vlq.MaxLen64) + for addrDesc, acs := range acm { + // address with 0 contracts is removed from db - happens on disconnect + if acs == nil || (acs.EthTxs == 0 && len(acs.Contracts) == 0) { + wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc)) + } else { + buf = buf[:0] + l := packVaruint(acs.EthTxs, varBuf) + buf = append(buf, varBuf[:l]...) + for _, ac := range acs.Contracts { + buf = append(buf, ac.Contract...) + l = packVaruint(ac.Txs, varBuf) + buf = append(buf, varBuf[:l]...) + } + wb.PutCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc), buf) + } + } + return nil +} + +// GetAddrDescContracts returns AddrContracts for given addrDesc +func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*AddrContracts, error) { + val, err := d.db.GetCF(d.ro, d.cfh[cfAddressContracts], addrDesc) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + if len(buf) == 0 { + return nil, nil + } + c := make([]AddrContract, 0, 4) + et, l := unpackVaruint(buf) + buf = buf[l:] + for len(buf) > 0 { + if len(buf) < eth.EthereumTypeAddressDescriptorLen { + return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String()) + } + txs, l := unpackVaruint(buf[eth.EthereumTypeAddressDescriptorLen:]) + contract := append(bchain.AddressDescriptor(nil), buf[:eth.EthereumTypeAddressDescriptorLen]...) + c = append(c, AddrContract{ + Contract: contract, + Txs: txs, + }) + buf = buf[eth.EthereumTypeAddressDescriptorLen+l:] + } + return &AddrContracts{EthTxs: et, Contracts: c}, nil +} + +func findContractInAddressContracts(contract bchain.AddressDescriptor, contracts []AddrContract) (int, bool) { + for i := range contracts { + if bytes.Equal(contract, contracts[i].Contract) { + return i, true + } + } + return 0, false +} + +func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) error { + var err error + strAddrDesc := string(addrDesc) + ac, e := addressContracts[strAddrDesc] + if !e { + ac, err = d.GetAddrDescContracts(addrDesc) + if err != nil { + return err + } + if ac == nil { + ac = &AddrContracts{} + } + addressContracts[strAddrDesc] = ac + d.cbs.balancesMiss++ + } else { + d.cbs.balancesHit++ + } + if contract == nil { + ac.EthTxs++ + } else { + // locate the contract and set i to the index in the array of contracts + i, found := findContractInAddressContracts(contract, ac.Contracts) + if !found { + i = len(ac.Contracts) + ac.Contracts = append(ac.Contracts, AddrContract{Contract: contract}) + } + // index 0 is for ETH transfers, contract indexes start with 1 + if index < 0 { + index = ^int32(i + 1) + } else { + index = int32(i + 1) + } + ac.Contracts[i].Txs++ + } + addresses[strAddrDesc] = append(addresses[strAddrDesc], outpoint{ + btxID: btxID, + index: index, + }) + return nil +} + +type ethBlockTxContract struct { + addr, contract bchain.AddressDescriptor +} + +type ethBlockTx struct { + btxID []byte + from, to bchain.AddressDescriptor + contracts []ethBlockTxContract +} + +func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses map[string][]outpoint, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) { + blockTxs := make([]ethBlockTx, len(block.Txs)) + for txi, tx := range block.Txs { + btxID, err := d.chainParser.PackTxid(tx.Txid) + if err != nil { + return nil, err + } + blockTx := &blockTxs[txi] + blockTx.btxID = btxID + // there is only one output address in EthereumType transaction, store it in format txid 0 + if len(tx.Vout) == 1 && len(tx.Vout[0].ScriptPubKey.Addresses) == 1 { + addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0]) + if err != nil { + // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output", err, block.Height, tx.Txid) + } + continue + } + if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, 0, nil, addresses, addressContracts); err != nil { + return nil, err + } + blockTx.to = addrDesc + } + // there is only one input address in EthereumType transaction, store it in format txid ^0 + if len(tx.Vin) == 1 && len(tx.Vin[0].Addresses) == 1 { + addrDesc, err := d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0]) + if err != nil { + if err != bchain.ErrAddressMissing { + glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, input", err, block.Height, tx.Txid) + } + continue + } + if err = d.addToAddressesAndContractsEthereumType(addrDesc, btxID, ^int32(0), nil, addresses, addressContracts); err != nil { + return nil, err + } + blockTx.from = addrDesc + } + // store erc20 transfers + erc20, err := d.chainParser.EthereumTypeGetErc20FromTx(&tx) + if err != nil { + glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v", err, block.Height, tx.Txid) + } + blockTx.contracts = make([]ethBlockTxContract, len(erc20)*2) + for i, t := range erc20 { + var contract, from, to bchain.AddressDescriptor + contract, err = d.chainParser.GetAddrDescFromAddress(t.Contract) + if err == nil { + from, err = d.chainParser.GetAddrDescFromAddress(t.From) + if err == nil { + to, err = d.chainParser.GetAddrDescFromAddress(t.To) + } + } + if err != nil { + glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v, transfer %v", err, block.Height, tx.Txid, t) + continue + } + if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts); err != nil { + return nil, err + } + bc := &blockTx.contracts[i*2] + bc.addr = from + bc.contract = contract + if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts); err != nil { + return nil, err + } + bc = &blockTx.contracts[i*2+1] + bc.addr = to + bc.contract = contract + } + } + return blockTxs, nil +} + +func (d *RocksDB) storeAndCleanupBlockTxsEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, blockTxs []ethBlockTx) error { + pl := d.chainParser.PackedTxidLen() + buf := make([]byte, 0, (pl+2*eth.EthereumTypeAddressDescriptorLen)*len(blockTxs)) + varBuf := make([]byte, vlq.MaxLen64) + zeroAddress := make([]byte, eth.EthereumTypeAddressDescriptorLen) + appendAddress := func(a bchain.AddressDescriptor) { + if len(a) != eth.EthereumTypeAddressDescriptorLen { + buf = append(buf, zeroAddress...) + } else { + buf = append(buf, a...) + } + } + for i := range blockTxs { + blockTx := &blockTxs[i] + buf = append(buf, blockTx.btxID...) + appendAddress(blockTx.from) + appendAddress(blockTx.to) + l := packVaruint(uint(len(blockTx.contracts)), varBuf) + buf = append(buf, varBuf[:l]...) + for j := range blockTx.contracts { + c := &blockTx.contracts[j] + appendAddress(c.addr) + appendAddress(c.contract) + } + } + key := packUint(block.Height) + wb.PutCF(d.cfh[cfBlockTxs], key, buf) + return d.cleanupBlockTxs(wb, block) +} + +func (d *RocksDB) getBlockTxsEthereumType(height uint32) ([]ethBlockTx, error) { + pl := d.chainParser.PackedTxidLen() + val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) + if err != nil { + return nil, err + } + defer val.Free() + buf := val.Data() + // nil data means the key was not found in DB + if buf == nil { + return nil, nil + } + // buf can be empty slice, this means the block did not contain any transactions + bt := make([]ethBlockTx, 0, 8) + getAddress := func(i int) (bchain.AddressDescriptor, int, error) { + if len(buf)-i < eth.EthereumTypeAddressDescriptorLen { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, 0, errors.New("Inconsistent data in blockTxs") + } + a := append(bchain.AddressDescriptor(nil), buf[i:i+eth.EthereumTypeAddressDescriptorLen]...) + // return null addresses as nil + for _, b := range a { + if b != 0 { + return a, i + eth.EthereumTypeAddressDescriptorLen, nil + } + } + return nil, i + eth.EthereumTypeAddressDescriptorLen, nil + } + var from, to bchain.AddressDescriptor + for i := 0; i < len(buf); { + if len(buf)-i < pl { + glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) + return nil, errors.New("Inconsistent data in blockTxs") + } + txid := append([]byte(nil), buf[i:i+pl]...) + i += pl + from, i, err = getAddress(i) + if err != nil { + return nil, err + } + to, i, err = getAddress(i) + if err != nil { + return nil, err + } + cc, l := unpackVaruint(buf[i:]) + i += l + contracts := make([]ethBlockTxContract, cc) + for j := range contracts { + contracts[j].addr, i, err = getAddress(i) + if err != nil { + return nil, err + } + contracts[j].contract, i, err = getAddress(i) + if err != nil { + return nil, err + } + } + bt = append(bt, ethBlockTx{ + btxID: txid, + from: from, + to: to, + contracts: contracts, + }) + } + return bt, nil +} + +func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, height uint32, blockTxs []ethBlockTx, contracts map[string]*AddrContracts) error { + glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") + addresses := make(map[string]struct{}) + disconnectAddress := func(btxID []byte, addrDesc, contract bchain.AddressDescriptor) error { + var err error + // do not process empty address + if len(addrDesc) == 0 { + return nil + } + s := string(addrDesc) + addresses[s] = struct{}{} + c, fc := contracts[s] + if !fc { + c, err = d.GetAddrDescContracts(addrDesc) + if err != nil { + return err + } + contracts[s] = c + } + if c != nil { + if contract == nil { + if c.EthTxs > 0 { + c.EthTxs-- + } else { + glog.Warning("AddressContracts ", addrDesc, ", EthTxs would be negative, tx ", hex.EncodeToString(btxID)) + } + } else { + i, found := findContractInAddressContracts(contract, c.Contracts) + if found { + if c.Contracts[i].Txs > 0 { + c.Contracts[i].Txs-- + if c.Contracts[i].Txs == 0 { + c.Contracts = append(c.Contracts[:i], c.Contracts[i+1:]...) + } + } else { + glog.Warning("AddressContracts ", addrDesc, ", contract ", i, " Txs would be negative, tx ", hex.EncodeToString(btxID)) + } + } else { + glog.Warning("AddressContracts ", addrDesc, ", contract ", contract, " not found, tx ", hex.EncodeToString(btxID)) + } + } + } else { + glog.Warning("AddressContracts ", addrDesc, " not found, tx ", hex.EncodeToString(btxID)) + } + return nil + } + for i := range blockTxs { + blockTx := &blockTxs[i] + if err := disconnectAddress(blockTx.btxID, blockTx.from, nil); err != nil { + return err + } + if err := disconnectAddress(blockTx.btxID, blockTx.to, nil); err != nil { + return err + } + for _, c := range blockTx.contracts { + if err := disconnectAddress(blockTx.btxID, c.addr, c.contract); err != nil { + return err + } + } + wb.DeleteCF(d.cfh[cfTransactions], blockTx.btxID) + } + for a := range addresses { + key := packAddressKey([]byte(a), height) + wb.DeleteCF(d.cfh[cfAddresses], key) + } + return nil +} + +// DisconnectBlockRangeEthereumType removes all data belonging to blocks in range lower-higher +// it is able to disconnect only blocks for which there are data in the blockTxs column +func (d *RocksDB) DisconnectBlockRangeEthereumType(lower uint32, higher uint32) error { + blocks := make([][]ethBlockTx, higher-lower+1) + for height := lower; height <= higher; height++ { + blockTxs, err := d.getBlockTxsEthereumType(height) + if err != nil { + return err + } + // nil blockTxs means blockTxs were not found in db + if blockTxs == nil { + return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) + } + blocks[height-lower] = blockTxs + } + wb := gorocksdb.NewWriteBatch() + defer wb.Destroy() + contracts := make(map[string]*AddrContracts) + for height := higher; height >= lower; height-- { + if err := d.disconnectBlockTxsEthereumType(wb, height, blocks[height-lower], contracts); err != nil { + return err + } + key := packUint(height) + wb.DeleteCF(d.cfh[cfBlockTxs], key) + wb.DeleteCF(d.cfh[cfHeight], key) + } + d.storeAddressContracts(wb, contracts) + err := d.db.Write(d.wo, wb) + if err == nil { + glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) + } + return err +} diff --git a/db/rocksdb_ethereumtype_test.go b/db/rocksdb_ethereumtype_test.go new file mode 100644 index 00000000..e57b192d --- /dev/null +++ b/db/rocksdb_ethereumtype_test.go @@ -0,0 +1,286 @@ +// build unittest + +package db + +import ( + "blockbook/bchain/coins/eth" + "blockbook/tests/dbtestdata" + "encoding/hex" + "reflect" + "testing" + + "github.com/juju/errors" +) + +type testEthereumParser struct { + *eth.EthereumParser +} + +func ethereumTestnetParser() *eth.EthereumParser { + return eth.NewEthereumParser(1) +} + +func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { + if err := checkColumn(d, cfHeight, []keyPair{ + keyPair{ + "0041eee8", + "c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839), + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } + // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values + if err := checkColumn(d, cfAddresses, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + if err := checkColumn(d, cfAddressContracts, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + var blockTxsKp []keyPair + if afterDisconnect { + blockTxsKp = []keyPair{} + } else { + blockTxsKp = []keyPair{ + keyPair{ + "0041eee8", + dbtestdata.EthTxidB1T1 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "00" + + dbtestdata.EthTxidB1T2 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + "02" + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), + nil, + }, + } + } + if err := checkColumn(d, cfBlockTxs, blockTxsKp); err != nil { + { + t.Fatal(err) + } + } +} + +func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) { + if err := checkColumn(d, cfHeight, []keyPair{ + keyPair{ + "0041eee8", + "c7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" + uintToHex(1534858022) + varuintToHex(2) + varuintToHex(31839), + nil, + }, + keyPair{ + "0041eee9", + "2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee" + uintToHex(1534859988) + varuintToHex(2) + varuintToHex(2345678), + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } + // the vout is encoded as signed varint, i.e. value * 2 for positive values, abs(value)*2 + 1 for negative values + if err := checkColumn(d, cfAddresses, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T1 + "00" + dbtestdata.EthTxidB1T2 + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "01" + dbtestdata.EthTxidB1T2 + "03", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "0041eee8", dbtestdata.EthTxidB1T2 + "00", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "01" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T1 + "00", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "01" + dbtestdata.EthTxidB2T2 + "02" + dbtestdata.EthTxidB2T2 + "05" + dbtestdata.EthTxidB2T2 + "04" + dbtestdata.EthTxidB2T2 + "03", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "03" + dbtestdata.EthTxidB2T2 + "04", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + "0041eee9", dbtestdata.EthTxidB2T2 + "00", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + if err := checkColumn(d, cfAddressContracts, []keyPair{ + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "00" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, + keyPair{dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "01", nil}, + }); err != nil { + { + t.Fatal(err) + } + } + + if err := checkColumn(d, cfBlockTxs, []keyPair{ + keyPair{ + "0041eee9", + dbtestdata.EthTxidB2T1 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser) + "00" + + dbtestdata.EthTxidB2T2 + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser) + + "08" + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser) + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser), + nil, + }, + }); err != nil { + { + t.Fatal(err) + } + } +} + +// TestRocksDB_Index_EthereumType is an integration test probing the whole indexing functionality for EthereumType chains +// It does the following: +// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block) +// 2) GetTransactions for various addresses / low-high ranges +// 3) GetBestBlock, GetBlockHash +// 4) Test tx caching functionality +// 5) Disconnect the block 2 using BlockTxs column +// 6) Reconnect block 2 and check +// After each step, the content of DB is examined and any difference against expected state is regarded as failure +func TestRocksDB_Index_EthereumType(t *testing.T) { + d := setupRocksDB(t, &testEthereumParser{ + EthereumParser: ethereumTestnetParser(), + }) + defer closeAndDestroyRocksDB(t, d) + + // connect 1st block + block1 := dbtestdata.GetTestEthereumTypeBlock1(d.chainParser) + if err := d.ConnectBlock(block1); err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock1(t, d, false) + + // connect 2nd block + block2 := dbtestdata.GetTestEthereumTypeBlock2(d.chainParser) + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock2(t, d) + + // get transactions for various addresses / low-high ranges + verifyGetTransactions(t, d, "0x"+dbtestdata.EthAddr55, 0, 10000000, []txidVoutOutput{ + txidVoutOutput{"0x" + dbtestdata.EthTxidB1T1, 0, true}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB1T2, 1, true}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB2T1, 0, false}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 2, false}, + txidVoutOutput{"0x" + dbtestdata.EthTxidB2T2, 1, true}, + }, nil) + verifyGetTransactions(t, d, "mtGXQvBowMkBpnhLckhxhbwYK44Gs9eBad", 500000, 1000000, []txidVoutOutput{}, errors.New("Address missing")) + + // GetBestBlock + height, hash, err := d.GetBestBlock() + if err != nil { + t.Fatal(err) + } + if height != 4321001 { + t.Fatalf("GetBestBlock: got height %v, expected %v", height, 4321001) + } + if hash != "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee" { + t.Fatalf("GetBestBlock: got hash %v, expected %v", hash, "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee") + } + + // GetBlockHash + hash, err = d.GetBlockHash(4321000) + if err != nil { + t.Fatal(err) + } + if hash != "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11" { + t.Fatalf("GetBlockHash: got hash %v, expected %v", hash, "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11") + } + + // Not connected block + hash, err = d.GetBlockHash(4321002) + if err != nil { + t.Fatal(err) + } + if hash != "" { + t.Fatalf("GetBlockHash: got hash '%v', expected ''", hash) + } + + // GetBlockHash + info, err := d.GetBlockInfo(4321001) + if err != nil { + t.Fatal(err) + } + iw := &BlockInfo{ + Hash: "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee", + Txs: 2, + Size: 2345678, + Time: 1534859988, + Height: 4321001, + } + if !reflect.DeepEqual(info, iw) { + t.Errorf("GetBlockInfo() = %+v, want %+v", info, iw) + } + + // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock + testTxCache(t, d, block1, &block1.Txs[0]) + testTxCache(t, d, block2, &block2.Txs[0]) + if err = d.PutTx(&block2.Txs[1], block2.Height, block2.Txs[1].Blocktime); err != nil { + t.Fatal(err) + } + if err = d.PutTx(&block2.Txs[1], block2.Height, block2.Txs[1].Blocktime); err != nil { + t.Fatal(err) + } + // check that there is only the last tx in the cache + packedTx, err := d.chainParser.PackTx(&block2.Txs[1], block2.Height, block2.Txs[1].Blocktime) + if err := checkColumn(d, cfTransactions, []keyPair{ + keyPair{dbtestdata.EthTxidB2T2, hex.EncodeToString(packedTx), nil}, + }); err != nil { + { + t.Fatal(err) + } + } + // try to disconnect both blocks, however only the last one is kept, it is not possible + err = d.DisconnectBlockRangeEthereumType(4321000, 4321001) + if err == nil || err.Error() != "Cannot disconnect blocks with height 4321000 and lower. It is necessary to rebuild index." { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock2(t, d) + + // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs + // and that the cached tx is removed + err = d.DisconnectBlockRangeEthereumType(4321001, 4321001) + if err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock1(t, d, true) + if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { + { + t.Fatal(err) + } + } + + // connect block again and verify the state of db + if err := d.ConnectBlock(block2); err != nil { + t.Fatal(err) + } + verifyAfterEthereumTypeBlock2(t, d) + +} diff --git a/db/rocksdb_test.go b/db/rocksdb_test.go index 771bde20..8367c483 100644 --- a/db/rocksdb_test.go +++ b/db/rocksdb_test.go @@ -1,4 +1,4 @@ -// +build unittest +// build unittest package db @@ -9,7 +9,6 @@ import ( "blockbook/tests/dbtestdata" "encoding/binary" "encoding/hex" - "fmt" "io/ioutil" "math/big" "os" @@ -33,6 +32,10 @@ func TestMain(m *testing.M) { os.Exit(c) } +type testBitcoinParser struct { + *btc.BitcoinParser +} + func bitcoinTestnetParser() *btc.BitcoinParser { return btc.NewBitcoinParser( btc.GetChainParams("test"), @@ -48,7 +51,7 @@ func setupRocksDB(t *testing.T, p bchain.BlockChainParser) *RocksDB { if err != nil { t.Fatal(err) } - is, err := d.LoadInternalState("btc-testnet") + is, err := d.LoadInternalState("coin-unittest") if err != nil { t.Fatal(err) } @@ -151,7 +154,7 @@ func checkColumn(d *RocksDB, col int, kp []keyPair) error { return nil } -func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { +func verifyAfterBitcoinTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{ "000370d5", @@ -232,7 +235,7 @@ func verifyAfterUTXOBlock1(t *testing.T, d *RocksDB, afterDisconnect bool) { } } -func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { +func verifyAfterBitcoinTypeBlock2(t *testing.T, d *RocksDB) { if err := checkColumn(d, cfHeight, []keyPair{ keyPair{ "000370d5", @@ -368,13 +371,13 @@ func verifyAfterUTXOBlock2(t *testing.T, d *RocksDB) { type txidVoutOutput struct { txid string - vout uint32 + vout int32 isOutput bool } func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint32, wantTxids []txidVoutOutput, wantErr error) { gotTxids := make([]txidVoutOutput, 0) - addToTxids := func(txid string, vout uint32, isOutput bool) error { + addToTxids := func(txid string, vout int32, isOutput bool) error { gotTxids = append(gotTxids, txidVoutOutput{txid, vout, isOutput}) return nil } @@ -388,10 +391,6 @@ func verifyGetTransactions(t *testing.T, d *RocksDB, addr string, low, high uint } } -type testBitcoinParser struct { - *btc.BitcoinParser -} - // override PackTx and UnpackTx to default BaseParser functionality // BitcoinParser uses tx hex which is not available for the test transactions func (p *testBitcoinParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { @@ -415,7 +414,7 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { } // Confirmations are not stored in the DB, set them from input tx gtx.Confirmations = tx.Confirmations - if fmt.Sprint(gtx) != fmt.Sprint(tx) { + if !reflect.DeepEqual(gtx, tx) { t.Errorf("GetTx: %v, want %v", gtx, tx) } if err := d.DeleteTx(tx.Txid); err != nil { @@ -423,35 +422,34 @@ func testTxCache(t *testing.T, d *RocksDB, b *bchain.Block, tx *bchain.Tx) { } } -// TestRocksDB_Index_UTXO is an integration test probing the whole indexing functionality for UTXO chains +// TestRocksDB_Index_BitcoinType is an integration test probing the whole indexing functionality for BitcoinType chains // It does the following: // 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block) // 2) GetTransactions for various addresses / low-high ranges // 3) GetBestBlock, GetBlockHash // 4) Test tx caching functionality -// 5) Disconnect block 2 - expect error -// 6) Disconnect the block 2 using BlockTxs column -// 7) Reconnect block 2 and check +// 5) Disconnect the block 2 using BlockTxs column +// 6) Reconnect block 2 and check // After each step, the content of DB is examined and any difference against expected state is regarded as failure -func TestRocksDB_Index_UTXO(t *testing.T) { +func TestRocksDB_Index_BitcoinType(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ BitcoinParser: bitcoinTestnetParser(), }) defer closeAndDestroyRocksDB(t, d) // connect 1st block - will log warnings about missing UTXO transactions in txAddresses column - block1 := dbtestdata.GetTestUTXOBlock1(d.chainParser) + block1 := dbtestdata.GetTestBitcoinTypeBlock1(d.chainParser) if err := d.ConnectBlock(block1); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, false) + verifyAfterBitcoinTypeBlock1(t, d, false) // connect 2nd block - use some outputs from the 1st block as the inputs and 1 input uses tx from the same block - block2 := dbtestdata.GetTestUTXOBlock2(d.chainParser) + block2 := dbtestdata.GetTestBitcoinTypeBlock2(d.chainParser) if err := d.ConnectBlock(block2); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock2(t, d) + verifyAfterBitcoinTypeBlock2(t, d) // get transactions for various addresses / low-high ranges verifyGetTransactions(t, d, dbtestdata.Addr2, 0, 1000000, []txidVoutOutput{ @@ -513,7 +511,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { Height: 225494, } if !reflect.DeepEqual(info, iw) { - t.Errorf("GetAddressBalance() = %+v, want %+v", info, iw) + t.Errorf("GetBlockInfo() = %+v, want %+v", info, iw) } // Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock @@ -532,27 +530,20 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } } - // DisconnectBlock for UTXO chains is not possible - err = d.DisconnectBlock(block2) - if err == nil || err.Error() != "DisconnectBlock is not supported for UTXO chains" { - t.Fatal(err) - } - verifyAfterUTXOBlock2(t, d) - // try to disconnect both blocks, however only the last one is kept, it is not possible - err = d.DisconnectBlockRangeUTXO(225493, 225494) + err = d.DisconnectBlockRangeBitcoinType(225493, 225494) if err == nil || err.Error() != "Cannot disconnect blocks with height 225493 and lower. It is necessary to rebuild index." { t.Fatal(err) } - verifyAfterUTXOBlock2(t, d) + verifyAfterBitcoinTypeBlock2(t, d) // disconnect the 2nd block, verify that the db contains only data from the 1st block with restored unspentTxs // and that the cached tx is removed - err = d.DisconnectBlockRangeUTXO(225494, 225494) + err = d.DisconnectBlockRangeBitcoinType(225494, 225494) if err != nil { t.Fatal(err) } - verifyAfterUTXOBlock1(t, d, true) + verifyAfterBitcoinTypeBlock1(t, d, true) if err := checkColumn(d, cfTransactions, []keyPair{}); err != nil { { t.Fatal(err) @@ -563,10 +554,9 @@ func TestRocksDB_Index_UTXO(t *testing.T) { if err := d.ConnectBlock(block2); err != nil { t.Fatal(err) } - verifyAfterUTXOBlock2(t, d) + verifyAfterBitcoinTypeBlock2(t, d) // test public methods for address balance and tx addresses - ab, err := d.GetAddressBalance(dbtestdata.Addr5) if err != nil { t.Fatal(err) @@ -627,7 +617,7 @@ func TestRocksDB_Index_UTXO(t *testing.T) { } -func Test_BulkConnect_UTXO(t *testing.T) { +func Test_BulkConnect_BitcoinType(t *testing.T) { d := setupRocksDB(t, &testBitcoinParser{ BitcoinParser: bitcoinTestnetParser(), }) @@ -642,7 +632,7 @@ func Test_BulkConnect_UTXO(t *testing.T) { t.Fatal("DB not in DbStateInconsistent") } - if err := bc.ConnectBlock(dbtestdata.GetTestUTXOBlock1(d.chainParser), false); err != nil { + if err := bc.ConnectBlock(dbtestdata.GetTestBitcoinTypeBlock1(d.chainParser), false); err != nil { t.Fatal(err) } if err := checkColumn(d, cfBlockTxs, []keyPair{}); err != nil { @@ -651,7 +641,7 @@ func Test_BulkConnect_UTXO(t *testing.T) { } } - if err := bc.ConnectBlock(dbtestdata.GetTestUTXOBlock2(d.chainParser), true); err != nil { + if err := bc.ConnectBlock(dbtestdata.GetTestBitcoinTypeBlock2(d.chainParser), true); err != nil { t.Fatal(err) } @@ -663,7 +653,7 @@ func Test_BulkConnect_UTXO(t *testing.T) { t.Fatal("DB not in DbStateOpen") } - verifyAfterUTXOBlock2(t, d) + verifyAfterBitcoinTypeBlock2(t, d) } func Test_packBigint_unpackBigint(t *testing.T) { @@ -848,17 +838,17 @@ func Test_packTxAddresses_unpackTxAddresses(t *testing.T) { Height: 123456789, Inputs: []TxInput{ { - AddrDesc: []byte{}, + AddrDesc: []byte(nil), ValueSat: *big.NewInt(1234), }, }, Outputs: []TxOutput{ { - AddrDesc: []byte{}, + AddrDesc: []byte(nil), ValueSat: *big.NewInt(5678), }, { - AddrDesc: []byte{}, + AddrDesc: []byte(nil), ValueSat: *big.NewInt(98), Spent: true, }, diff --git a/db/sync.go b/db/sync.go index 6bae828f..e755c027 100644 --- a/db/sync.go +++ b/db/sync.go @@ -393,26 +393,11 @@ func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { // DisconnectBlocks removes all data belonging to blocks in range lower-higher, func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) - // if the chain is UTXO, always use DisconnectBlockRange - if w.chain.GetChainParser().IsUTXOChain() { - return w.db.DisconnectBlockRangeUTXO(lower, higher) + ct := w.chain.GetChainParser().GetChainType() + if ct == bchain.ChainBitcoinType { + return w.db.DisconnectBlockRangeBitcoinType(lower, higher) + } else if ct == bchain.ChainEthereumType { + return w.db.DisconnectBlockRangeEthereumType(lower, higher) } - blocks := make([]*bchain.Block, len(hashes)) - var err error - // try to get all blocks first to see if we can avoid full scan - for i, hash := range hashes { - blocks[i], err = w.chain.GetBlock(hash, 0) - if err != nil { - // cannot get a block, we must do full range scan - return w.db.DisconnectBlockRangeNonUTXO(lower, higher) - } - } - // got all blocks to be disconnected, disconnect them one after another - for i, block := range blocks { - glog.Info("Disconnecting block ", (int(higher) - i), " ", block.Hash) - if err = w.db.DisconnectBlock(block); err != nil { - return err - } - } - return nil + return errors.New("Unknown chain type") } diff --git a/db/txcache.go b/db/txcache.go index 25f17bff..b20d21fe 100644 --- a/db/txcache.go +++ b/db/txcache.go @@ -2,18 +2,21 @@ package db import ( "blockbook/bchain" + "blockbook/bchain/coins/eth" "blockbook/common" "github.com/golang/glog" + "github.com/juju/errors" ) // TxCache is handle to TxCacheServer type TxCache struct { - db *RocksDB - chain bchain.BlockChain - metrics *common.Metrics - is *common.InternalState - enabled bool + db *RocksDB + chain bchain.BlockChain + metrics *common.Metrics + is *common.InternalState + enabled bool + chainType bchain.ChainType } // NewTxCache creates new TxCache interface and returns its handle @@ -22,11 +25,12 @@ func NewTxCache(db *RocksDB, chain bchain.BlockChain, metrics *common.Metrics, i glog.Info("txcache: disabled") } return &TxCache{ - db: db, - chain: chain, - metrics: metrics, - is: is, - enabled: enabled, + db: db, + chain: chain, + metrics: metrics, + is: is, + enabled: enabled, + chainType: chain.GetChainParser().GetChainType(), }, nil } @@ -56,18 +60,27 @@ func (c *TxCache) GetTransaction(txid string) (*bchain.Tx, uint32, error) { c.metrics.TxCacheEfficiency.With(common.Labels{"status": "miss"}).Inc() // cache only confirmed transactions if tx.Confirmations > 0 { - ta, err := c.db.GetTxAddresses(txid) - if err != nil { - return nil, 0, err - } - // the transaction may me not yet indexed, in that case get the height from the backend - if ta == nil { - h, err = c.chain.GetBestBlockHeight() + if c.chainType == bchain.ChainBitcoinType { + ta, err := c.db.GetTxAddresses(txid) + if err != nil { + return nil, 0, err + } + // the transaction may me not yet indexed, in that case get the height from the backend + if ta == nil { + h, err = c.chain.GetBestBlockHeight() + if err != nil { + return nil, 0, err + } + } else { + h = ta.Height + } + } else if c.chainType == bchain.ChainEthereumType { + h, err = eth.GetHeightFromTx(tx) if err != nil { return nil, 0, err } } else { - h = ta.Height + return nil, 0, errors.New("Unknown chain type") } if c.enabled { err = c.db.PutTx(tx, h, tx.Blocktime) diff --git a/docs/config.md b/docs/config.md index 509edd8c..9c796b16 100644 --- a/docs/config.md +++ b/docs/config.md @@ -86,8 +86,8 @@ Good examples of coin configuration are * `parse` – Use binary parser for block decoding if *true* else call verbose back-end RPC method that returns JSON. Note that verbose method is slow and not every coin support it. However there are coin implementations that don't support binary parsing (e.g. ZCash). - * `mempool_workers` – Number of workers for UTXO mempool. - * `mempool_sub_workers` – Number of subworkers for UTXO mempool. + * `mempool_workers` – Number of workers for BitcoinType mempool. + * `mempool_sub_workers` – Number of subworkers for BitcoinType mempool. * `block_addresses_to_keep` – Number of blocks that are to be kept in blockaddresses column. * `additional_params` – Object of coin-specific params. diff --git a/server/public.go b/server/public.go index f0a7c795..363b57e1 100644 --- a/server/public.go +++ b/server/public.go @@ -14,6 +14,7 @@ import ( "net/http" "reflect" "runtime" + "runtime/debug" "strconv" "strings" "time" @@ -25,11 +26,18 @@ const txsOnPage = 25 const blocksOnPage = 50 const txsInAPI = 1000 +const ( + _ = iota + apiV1 + apiV2 +) + // PublicServer is a handle to public http server type PublicServer struct { binding string certFiles string socketio *SocketIoServer + websocket *WebsocketServer https *http.Server db *db.RocksDB txCache *db.TxCache @@ -58,6 +66,11 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch return nil, err } + websocket, err := NewWebsocketServer(db, chain, txCache, metrics, is) + if err != nil { + return nil, err + } + addr, path := splitBinding(binding) serveMux := http.NewServeMux() https := &http.Server{ @@ -71,6 +84,7 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch https: https, api: api, socketio: socketio, + websocket: websocket, db: db, txCache: txCache, chain: chain, @@ -81,7 +95,7 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch is: is, debug: debugMode, } - s.templates = parseTemplates() + s.templates = s.parseTemplates() // map only basic functions, the rest is enabled by method MapFullPublicInterface serveMux.Handle(path+"favicon.ico", http.FileServer(http.Dir("./static/"))) @@ -89,7 +103,7 @@ func NewPublicServer(binding string, certFiles string, db *db.RocksDB, chain bch // default handler serveMux.HandleFunc(path, s.htmlTemplateHandler(s.explorerIndex)) // default API handler - serveMux.HandleFunc(path+"api/", s.jsonHandler(s.apiIndex)) + serveMux.HandleFunc(path+"api/", s.jsonHandler(s.apiIndex, apiV2)) return s, nil } @@ -108,8 +122,9 @@ func (s *PublicServer) Run() error { func (s *PublicServer) ConnectFullPublicInterface() { serveMux := s.https.Handler.(*http.ServeMux) _, path := splitBinding(s.binding) - // support for tests of socket.io interface - serveMux.Handle(path+"test.html", http.FileServer(http.Dir("./static/"))) + // support for test pages + serveMux.Handle(path+"test-socketio.html", http.FileServer(http.Dir("./static/"))) + serveMux.Handle(path+"test-websocket.html", http.FileServer(http.Dir("./static/"))) if s.internalExplorer { // internal explorer handlers serveMux.HandleFunc(path+"tx/", s.htmlTemplateHandler(s.explorerTx)) @@ -125,16 +140,46 @@ func (s *PublicServer) ConnectFullPublicInterface() { serveMux.HandleFunc(path+"address/", s.addressRedirect) } // API calls - serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex)) - serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx)) - serveMux.HandleFunc(path+"api/tx-specific/", s.jsonHandler(s.apiTxSpecific)) - serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress)) - serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo)) - serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock)) - serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx)) - serveMux.HandleFunc(path+"api/estimatefee/", s.jsonHandler(s.apiEstimateFee)) + // default api without version can be changed to different version at any time + // use versioned api for stability + + var apiDefault int + // ethereum supports only api V2 + if s.chainParser.GetChainType() == bchain.ChainEthereumType { + apiDefault = apiV2 + } else { + apiDefault = apiV1 + // legacy v1 format + serveMux.HandleFunc(path+"api/v1/block-index/", s.jsonHandler(s.apiBlockIndex, apiV1)) + serveMux.HandleFunc(path+"api/v1/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV1)) + serveMux.HandleFunc(path+"api/v1/tx/", s.jsonHandler(s.apiTx, apiV1)) + serveMux.HandleFunc(path+"api/v1/address/", s.jsonHandler(s.apiAddress, apiV1)) + serveMux.HandleFunc(path+"api/v1/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV1)) + serveMux.HandleFunc(path+"api/v1/block/", s.jsonHandler(s.apiBlock, apiV1)) + serveMux.HandleFunc(path+"api/v1/sendtx/", s.jsonHandler(s.apiSendTx, apiV1)) + serveMux.HandleFunc(path+"api/v1/estimatefee/", s.jsonHandler(s.apiEstimateFee, apiV1)) + } + serveMux.HandleFunc(path+"api/block-index/", s.jsonHandler(s.apiBlockIndex, apiDefault)) + serveMux.HandleFunc(path+"api/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiDefault)) + serveMux.HandleFunc(path+"api/tx/", s.jsonHandler(s.apiTx, apiDefault)) + serveMux.HandleFunc(path+"api/address/", s.jsonHandler(s.apiAddress, apiDefault)) + serveMux.HandleFunc(path+"api/utxo/", s.jsonHandler(s.apiAddressUtxo, apiDefault)) + serveMux.HandleFunc(path+"api/block/", s.jsonHandler(s.apiBlock, apiDefault)) + serveMux.HandleFunc(path+"api/sendtx/", s.jsonHandler(s.apiSendTx, apiDefault)) + serveMux.HandleFunc(path+"api/estimatefee/", s.jsonHandler(s.apiEstimateFee, apiDefault)) + // v2 format + serveMux.HandleFunc(path+"api/v2/block-index/", s.jsonHandler(s.apiBlockIndex, apiV2)) + serveMux.HandleFunc(path+"api/v2/tx-specific/", s.jsonHandler(s.apiTxSpecific, apiV2)) + serveMux.HandleFunc(path+"api/v2/tx/", s.jsonHandler(s.apiTx, apiV2)) + serveMux.HandleFunc(path+"api/v2/address/", s.jsonHandler(s.apiAddress, apiV2)) + serveMux.HandleFunc(path+"api/v2/utxo/", s.jsonHandler(s.apiAddressUtxo, apiV2)) + serveMux.HandleFunc(path+"api/v2/block/", s.jsonHandler(s.apiBlock, apiV2)) + serveMux.HandleFunc(path+"api/v2/sendtx/", s.jsonHandler(s.apiSendTx, apiV2)) + serveMux.HandleFunc(path+"api/v2/estimatefee/", s.jsonHandler(s.apiEstimateFee, apiV2)) // socket.io interface serveMux.Handle(path+"socket.io/", s.socketio.GetHandler()) + // websocket interface + serveMux.Handle(path+"websocket", s.websocket.GetHandler()) } // Close closes the server @@ -152,11 +197,13 @@ func (s *PublicServer) Shutdown(ctx context.Context) error { // OnNewBlock notifies users subscribed to bitcoind/hashblock about new block func (s *PublicServer) OnNewBlock(hash string, height uint32) { s.socketio.OnNewBlockHash(hash) + s.websocket.OnNewBlock(hash, height) } // OnNewTxAddr notifies users subscribed to bitcoind/addresstxid about new block -func (s *PublicServer) OnNewTxAddr(txid string, desc bchain.AddressDescriptor, isOutput bool) { - s.socketio.OnNewTxAddr(txid, desc, isOutput) +func (s *PublicServer) OnNewTxAddr(tx *bchain.Tx, desc bchain.AddressDescriptor) { + s.socketio.OnNewTxAddr(tx.Txid, desc) + s.websocket.OnNewTxAddr(tx, desc) } func (s *PublicServer) txRedirect(w http.ResponseWriter, r *http.Request) { @@ -191,7 +238,7 @@ func getFunctionName(i interface{}) string { return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() } -func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, error)) func(w http.ResponseWriter, r *http.Request) { +func (s *PublicServer) jsonHandler(handler func(r *http.Request, apiVersion int) (interface{}, error), apiVersion int) func(w http.ResponseWriter, r *http.Request) { type jsonError struct { Text string `json:"error"` HTTPStatus int `json:"-"` @@ -202,6 +249,7 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, e defer func() { if e := recover(); e != nil { glog.Error(getFunctionName(handler), " recovered from panic: ", e) + debug.PrintStack() if s.debug { data = jsonError{fmt.Sprint("Internal server error: recovered from panic ", e), http.StatusInternalServerError} } else { @@ -214,7 +262,7 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request) (interface{}, e } json.NewEncoder(w).Encode(data) }() - data, err = handler(r) + data, err = handler(r, apiVersion) if err != nil || data == nil { if apiErr, ok := err.(*api.APIError); ok { if apiErr.Public { @@ -245,6 +293,7 @@ func (s *PublicServer) newTemplateData() *TemplateData { CoinName: s.is.Coin, CoinShortcut: s.is.CoinShortcut, CoinLabel: s.is.CoinLabel, + ChainType: s.chainParser.GetChainType(), InternalExplorer: s.internalExplorer && !s.is.InitialSync, TOSLink: api.Text.TOSLink, } @@ -264,6 +313,7 @@ func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r defer func() { if e := recover(); e != nil { glog.Error(getFunctionName(handler), " recovered from panic: ", e) + debug.PrintStack() t = errorInternalTpl if s.debug { data = s.newTemplateDataWithError(fmt.Sprint("Internal server error: recovered from panic ", e)) @@ -286,7 +336,7 @@ func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r if s.debug { // reload templates on each request // to reflect changes during development - s.templates = parseTemplates() + s.templates = s.parseTemplates() } t, data, err = handler(w, r) if err != nil || (data == nil && t != noTpl) { @@ -333,10 +383,10 @@ type TemplateData struct { CoinShortcut string CoinLabel string InternalExplorer bool + ChainType bchain.ChainType Address *api.Address AddrStr string Tx *api.Tx - TxSpecific json.RawMessage Error *api.APIError Blocks *api.Blocks Block *api.Block @@ -345,28 +395,36 @@ type TemplateData struct { PrevPage int NextPage int PagingRange []int + PageParams template.URL TOSLink string SendTxHex string Status string } -func parseTemplates() []*template.Template { +func (s *PublicServer) parseTemplates() []*template.Template { templateFuncMap := template.FuncMap{ - "formatTime": formatTime, - "formatUnixTime": formatUnixTime, - "formatAmount": formatAmount, - "setTxToTemplateData": setTxToTemplateData, - "stringInSlice": stringInSlice, + "formatTime": formatTime, + "formatUnixTime": formatUnixTime, + "formatAmount": s.formatAmount, + "formatAmountWithDecimals": formatAmountWithDecimals, + "setTxToTemplateData": setTxToTemplateData, + "stringInSlice": stringInSlice, } t := make([]*template.Template, tplCount) t[errorTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) t[errorInternalTpl] = template.Must(template.New("error").Funcs(templateFuncMap).ParseFiles("./static/templates/error.html", "./static/templates/base.html")) t[indexTpl] = template.Must(template.New("index").Funcs(templateFuncMap).ParseFiles("./static/templates/index.html", "./static/templates/base.html")) - t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) - t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) t[blocksTpl] = template.Must(template.New("blocks").Funcs(templateFuncMap).ParseFiles("./static/templates/blocks.html", "./static/templates/paging.html", "./static/templates/base.html")) - t[blockTpl] = template.Must(template.New("block").Funcs(templateFuncMap).ParseFiles("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) t[sendTransactionTpl] = template.Must(template.New("block").Funcs(templateFuncMap).ParseFiles("./static/templates/sendtx.html", "./static/templates/base.html")) + if s.chainParser.GetChainType() == bchain.ChainEthereumType { + t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail_ethereumtype.html", "./static/templates/base.html")) + t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail_ethereumtype.html", "./static/templates/paging.html", "./static/templates/base.html")) + t[blockTpl] = template.Must(template.New("block").Funcs(templateFuncMap).ParseFiles("./static/templates/block.html", "./static/templates/txdetail_ethereumtype.html", "./static/templates/paging.html", "./static/templates/base.html")) + } else { + t[txTpl] = template.Must(template.New("tx").Funcs(templateFuncMap).ParseFiles("./static/templates/tx.html", "./static/templates/txdetail.html", "./static/templates/base.html")) + t[addressTpl] = template.Must(template.New("address").Funcs(templateFuncMap).ParseFiles("./static/templates/address.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) + t[blockTpl] = template.Must(template.New("block").Funcs(templateFuncMap).ParseFiles("./static/templates/block.html", "./static/templates/txdetail.html", "./static/templates/paging.html", "./static/templates/base.html")) + } return t } @@ -380,11 +438,18 @@ func formatTime(t time.Time) string { // for now return the string as it is // in future could be used to do coin specific formatting -func formatAmount(a string) string { - if a == "" { +func (s *PublicServer) formatAmount(a *api.Amount) string { + if a == nil { return "0" } - return a + return s.chainParser.AmountToDecimalString((*big.Int)(a)) +} + +func formatAmountWithDecimals(a *api.Amount, d int) string { + if a == nil { + return "0" + } + return a.DecimalString(d) } // called from template to support txdetail.html functionality @@ -395,23 +460,17 @@ func setTxToTemplateData(td *TemplateData, tx *api.Tx) *TemplateData { func (s *PublicServer) explorerTx(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var tx *api.Tx - var txSpecific json.RawMessage var err error s.metrics.ExplorerViews.With(common.Labels{"action": "tx"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] - tx, err = s.api.GetTransaction(txid, false) - if err != nil { - return errorTpl, nil, err - } - txSpecific, err = s.chain.GetTransactionSpecific(txid) + tx, err = s.api.GetTransaction(txid, false, true) if err != nil { return errorTpl, nil, err } } data := s.newTemplateData() data.Tx = tx - data.TxSpecific = txSpecific return txTpl, data, nil } @@ -438,6 +497,8 @@ func (s *PublicServer) explorerSpendingTx(w http.ResponseWriter, r *http.Request func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error) { var address *api.Address + var filter string + var fn = api.AddressFilterVoutOff var err error s.metrics.ExplorerViews.With(common.Labels{"action": "address"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { @@ -445,7 +506,21 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( if ec != nil { page = 0 } - address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsOnPage, false) + filter = r.URL.Query().Get("filter") + if len(filter) > 0 { + if filter == "inputs" { + fn = api.AddressFilterVoutInputs + } else if filter == "outputs" { + fn = api.AddressFilterVoutOutputs + } else { + fn, ec = strconv.Atoi(filter) + if ec != nil || fn < 0 { + filter = "" + fn = api.AddressFilterVoutOff + } + } + } + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsOnPage, api.TxHistoryLight, &api.AddressFilter{Vout: fn}) if err != nil { return errorTpl, nil, err } @@ -455,6 +530,10 @@ func (s *PublicServer) explorerAddress(w http.ResponseWriter, r *http.Request) ( data.Address = address data.Page = address.Page data.PagingRange, data.PrevPage, data.NextPage = getPagingRange(address.Page, address.TotalPages) + if filter != "" { + data.PageParams = template.URL("&filter=" + filter) + data.Address.Filter = filter + } return addressTpl, data, nil } @@ -524,12 +603,12 @@ func (s *PublicServer) explorerSearch(w http.ResponseWriter, r *http.Request) (t http.Redirect(w, r, joinURL("/block/", block.Hash), 302) return noTpl, nil, nil } - tx, err = s.api.GetTransaction(q, false) + tx, err = s.api.GetTransaction(q, false, false) if err == nil { http.Redirect(w, r, joinURL("/tx/", tx.Txid), 302) return noTpl, nil, nil } - address, err = s.api.GetAddress(q, 0, 1, true) + address, err = s.api.GetAddress(q, 0, 1, api.Basic, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) if err == nil { http.Redirect(w, r, joinURL("/address/", address.AddrStr), 302) return noTpl, nil, nil @@ -614,12 +693,12 @@ func getPagingRange(page int, total int) ([]int, int, int) { return r, pp, np } -func (s *PublicServer) apiIndex(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiIndex(r *http.Request, apiVersion int) (interface{}, error) { s.metrics.ExplorerViews.With(common.Labels{"action": "api-index"}).Inc() return s.api.GetSystemInfo(false) } -func (s *PublicServer) apiBlockIndex(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiBlockIndex(r *http.Request, apiVersion int) (interface{}, error) { type resBlockIndex struct { BlockHash string `json:"blockHash"` } @@ -645,7 +724,7 @@ func (s *PublicServer) apiBlockIndex(r *http.Request) (interface{}, error) { }, nil } -func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiTx(r *http.Request, apiVersion int) (interface{}, error) { var tx *api.Tx var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-tx"}).Inc() @@ -659,23 +738,26 @@ func (s *PublicServer) apiTx(r *http.Request) (interface{}, error) { return nil, api.NewAPIError("Parameter 'spending' cannot be converted to boolean", true) } } - tx, err = s.api.GetTransaction(txid, spendingTxs) + tx, err = s.api.GetTransaction(txid, spendingTxs, false) + if err == nil && apiVersion == apiV1 { + return s.api.TxToV1(tx), nil + } } return tx, err } -func (s *PublicServer) apiTxSpecific(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiTxSpecific(r *http.Request, apiVersion int) (interface{}, error) { var tx json.RawMessage var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-tx-specific"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { txid := r.URL.Path[i+1:] - tx, err = s.chain.GetTransactionSpecific(txid) + tx, err = s.chain.GetTransactionSpecific(&bchain.Tx{Txid: txid}) } return tx, err } -func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiAddress(r *http.Request, apiVersion int) (interface{}, error) { var address *api.Address var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-address"}).Inc() @@ -684,12 +766,15 @@ func (s *PublicServer) apiAddress(r *http.Request) (interface{}, error) { if ec != nil { page = 0 } - address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsInAPI, true) + address, err = s.api.GetAddress(r.URL.Path[i+1:], page, txsInAPI, api.TxidHistory, &api.AddressFilter{Vout: api.AddressFilterVoutOff}) + if err == nil && apiVersion == apiV1 { + return s.api.AddressToV1(address), nil + } } return address, err } -func (s *PublicServer) apiAddressUtxo(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiAddressUtxo(r *http.Request, apiVersion int) (interface{}, error) { var utxo []api.AddressUtxo var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-address"}).Inc() @@ -703,11 +788,14 @@ func (s *PublicServer) apiAddressUtxo(r *http.Request) (interface{}, error) { } } utxo, err = s.api.GetAddressUtxo(r.URL.Path[i+1:], onlyConfirmed) + if err == nil && apiVersion == apiV1 { + return s.api.AddressUtxoToV1(utxo), nil + } } return utxo, err } -func (s *PublicServer) apiBlock(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiBlock(r *http.Request, apiVersion int) (interface{}, error) { var block *api.Block var err error s.metrics.ExplorerViews.With(common.Labels{"action": "api-block"}).Inc() @@ -717,6 +805,9 @@ func (s *PublicServer) apiBlock(r *http.Request) (interface{}, error) { page = 0 } block, err = s.api.GetBlock(r.URL.Path[i+1:], page, txsInAPI) + if err == nil && apiVersion == apiV1 { + return s.api.BlockToV1(block), nil + } } return block, err } @@ -725,7 +816,7 @@ type resultSendTransaction struct { Result string `json:"result"` } -func (s *PublicServer) apiSendTx(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiSendTx(r *http.Request, apiVersion int) (interface{}, error) { var err error var res resultSendTransaction var hex string @@ -755,7 +846,7 @@ type resultEstimateFeeAsString struct { Result string `json:"result"` } -func (s *PublicServer) apiEstimateFee(r *http.Request) (interface{}, error) { +func (s *PublicServer) apiEstimateFee(r *http.Request, apiVersion int) (interface{}, error) { var res resultEstimateFeeAsString s.metrics.ExplorerViews.With(common.Labels{"action": "api-estimatefee"}).Inc() if i := strings.LastIndexByte(r.URL.Path, '/'); i > 0 { diff --git a/server/public_test.go b/server/public_test.go index f2f2a4ba..ee693c30 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -48,10 +48,10 @@ func setupRocksDB(t *testing.T, parser bchain.BlockChainParser) (*db.RocksDB, *c } d.SetInternalState(is) // import data - if err := d.ConnectBlock(dbtestdata.GetTestUTXOBlock1(parser)); err != nil { + if err := d.ConnectBlock(dbtestdata.GetTestBitcoinTypeBlock1(parser)); err != nil { t.Fatal(err) } - if err := d.ConnectBlock(dbtestdata.GetTestUTXOBlock2(parser)); err != nil { + if err := d.ConnectBlock(dbtestdata.GetTestBitcoinTypeBlock2(parser)); err != nil { t.Fatal(err) } return d, is, tmp @@ -131,7 +131,7 @@ func newPostRequest(u string, body string) *http.Request { return r } -func httpTests(t *testing.T, ts *httptest.Server) { +func httpTests_BitcoinType(t *testing.T, ts *httptest.Server) { tests := []struct { name string r *http.Request @@ -359,6 +359,7 @@ func httpTests(t *testing.T, ts *httptest.Server) { body: []string{ `{"blockbook":{"coin":"Fakecoin"`, `"bestHeight":225494`, + `"decimals":8`, `"backend":{"chain":"fakecoin","blocks":2,"headers":2,"bestblockhash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6"`, `"version":"001001","subversion":"/Fakecoin:0.0.1/"`, }, @@ -373,17 +374,35 @@ func httpTests(t *testing.T, ts *httptest.Server) { }, }, { - name: "apiTx", - r: newGetRequest(ts.URL + "/api/tx/05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07"), + name: "apiTx v1", + r: newGetRequest(ts.URL + "/api/v1/tx/05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07"), status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ - `{"txid":"05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vout":2,"n":0,"scriptSig":{"hex":""},"addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"],"value":"0.00009876"}],"vout":[{"value":"0.00009","n":0,"scriptPubKey":{"hex":"a914e921fc4912a315078f370d959f2c4f7b6d2a683c87","addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"]},"spent":false}],"blockhash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockheight":225494,"confirmations":1,"time":22549400002,"blocktime":22549400002,"valueOut":"0.00009","valueIn":"0.00009876","fees":"0.00000876","hex":""}`, + `{"txid":"05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vout":2,"n":0,"scriptSig":{},"addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"],"value":"0.00009876"}],"vout":[{"value":"0.00009","n":0,"scriptPubKey":{"hex":"a914e921fc4912a315078f370d959f2c4f7b6d2a683c87","addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"]},"spent":false}],"blockhash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockheight":225494,"confirmations":1,"time":22549400002,"blocktime":22549400002,"valueOut":"0.00009","valueIn":"0.00009876","fees":"0.00000876","hex":""}`, }, }, { - name: "apiTx - not found", - r: newGetRequest(ts.URL + "/api/tx/1232e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07"), + name: "apiTx - not found v1", + r: newGetRequest(ts.URL + "/api/v1/tx/1232e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07"), + status: http.StatusBadRequest, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"error":"Tx not found, Not found"}`, + }, + }, + { + name: "apiTx v2", + r: newGetRequest(ts.URL + "/api/v2/tx/05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"txid":"05e2e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07","vin":[{"txid":"effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75","vout":2,"n":0,"addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"],"value":"9876"}],"vout":[{"value":"9000","n":0,"hex":"a914e921fc4912a315078f370d959f2c4f7b6d2a683c87","addresses":["2NEVv9LJmAnY99W1pFoc5UJjVdypBqdnvu1"]}],"blockhash":"00000000eb0443fd7dc4a1ed5c686a8e995057805f9a161d9a5a77a95e72b7b6","blockheight":225494,"confirmations":1,"time":22549400002,"blocktime":22549400002,"value":"9000","valueIn":"9876","fees":"876"}`, + }, + }, + { + name: "apiTx - not found v2", + r: newGetRequest(ts.URL + "/api/v2/tx/1232e48aeabdd9b75def7b48d756ba304713c2aba7b522bf9dbc893fc4231b07"), status: http.StatusBadRequest, contentType: "application/json; charset=utf-8", body: []string{ @@ -396,12 +415,12 @@ func httpTests(t *testing.T, ts *httptest.Server) { status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ - `{"hex":"","txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","version":0,"locktime":0,"vin":null,"vout":[{"ValueSat":100000000,"value":0,"n":0,"scriptPubKey":{"hex":"76a914010d39800f86122416e28f485029acf77507169288ac","addresses":null}},{"ValueSat":12345,"value":0,"n":1,"scriptPubKey":{"hex":"76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac","addresses":null}}],"confirmations":2,"time":22549300000,"blocktime":22549300000}`, + `{"hex":"","txid":"00b2c06055e5e90e9c82bd4181fde310104391a7fa4f289b1704e5d90caa3840","version":0,"locktime":0,"vin":[],"vout":[{"ValueSat":100000000,"value":0,"n":0,"scriptPubKey":{"hex":"76a914010d39800f86122416e28f485029acf77507169288ac","addresses":null}},{"ValueSat":12345,"value":0,"n":1,"scriptPubKey":{"hex":"76a9148bdf0aa3c567aa5975c2e61321b8bebbe7293df688ac","addresses":null}}],"confirmations":2,"time":22549300000,"blocktime":22549300000}`, }, }, { - name: "apiAddress", - r: newGetRequest(ts.URL + "/api/address/mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"), + name: "apiAddress v1", + r: newGetRequest(ts.URL + "/api/v1/address/mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"), status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ @@ -409,14 +428,32 @@ func httpTests(t *testing.T, ts *httptest.Server) { }, }, { - name: "apiAddressUtxo", - r: newGetRequest(ts.URL + "/api/utxo/mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"), + name: "apiAddress v2", + r: newGetRequest(ts.URL + "/api/v2/address/mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `{"page":1,"totalPages":1,"itemsOnPage":1000,"addrStr":"mv9uLThosiEnGRbVPS7Vhyw6VssbVRsiAw","balance":"0","totalReceived":"1234567890123","totalSent":"1234567890123","unconfirmedBalance":"0","unconfirmedTxApperances":0,"txApperances":2,"txids":["7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","effd9ef509383d536b1c8af5bf434c8efbf521a4f2befd4022bbd68694b4ac75"]}`, + }, + }, + { + name: "apiAddressUtxo v1", + r: newGetRequest(ts.URL + "/api/v1/utxo/mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"), status: http.StatusOK, contentType: "application/json; charset=utf-8", body: []string{ `[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vout":1,"amount":"9172.83951061","satoshis":917283951061,"height":225494,"confirmations":1}]`, }, }, + { + name: "apiAddressUtxo v2", + r: newGetRequest(ts.URL + "/api/v2/utxo/mtR97eM2HPWVM6c8FGLGcukgaHHQv7THoL"), + status: http.StatusOK, + contentType: "application/json; charset=utf-8", + body: []string{ + `[{"txid":"7c3be24063f268aaa1ed81b64776798f56088757641a34fb156c4f51ed2e9d25","vout":1,"value":"917283951061","height":225494,"confirmations":1}]`, + }, + }, { name: "apiSendTx", r: newGetRequest(ts.URL + "/api/sendtx/1234567890"), @@ -475,7 +512,7 @@ func httpTests(t *testing.T, ts *httptest.Server) { b := string(bb) for _, c := range tt.body { if !strings.Contains(b, c) { - t.Errorf("Page body does not contain %v, body %v", c, b) + t.Errorf("got %v, want to contain %v", b, c) break } } @@ -483,7 +520,7 @@ func httpTests(t *testing.T, ts *httptest.Server) { } } -func socketioTests(t *testing.T, ts *httptest.Server) { +func socketioTests_BitcoinType(t *testing.T, ts *httptest.Server) { type socketioReq struct { Method string `json:"method"` Params []interface{} `json:"params"` @@ -578,13 +615,13 @@ func socketioTests(t *testing.T, ts *httptest.Server) { t.Errorf("Socketio error %v", err) } if resp != tt.want { - t.Errorf("Socketio resp %v, want %v", resp, tt.want) + t.Errorf("got %v, want %v", resp, tt.want) } }) } } -func Test_PublicServer_UTXO(t *testing.T) { +func Test_PublicServer_BitcoinType(t *testing.T) { s, dbpath := setupPublicHTTPServer(t) defer closeAndDestroyPublicServer(t, s, dbpath) s.ConnectFullPublicInterface() @@ -592,7 +629,6 @@ func Test_PublicServer_UTXO(t *testing.T) { ts := httptest.NewServer(s.https.Handler) defer ts.Close() - httpTests(t, ts) - socketioTests(t, ts) - + httpTests_BitcoinType(t, ts) + socketioTests_BitcoinType(t, ts) } diff --git a/server/socketio.go b/server/socketio.go index 9c7632d3..20022027 100644 --- a/server/socketio.go +++ b/server/socketio.go @@ -8,6 +8,7 @@ import ( "encoding/json" "math/big" "net/http" + "runtime/debug" "strconv" "strings" "time" @@ -161,6 +162,7 @@ func (s *SocketIoServer) onMessage(c *gosocketio.Channel, req map[string]json.Ra defer func() { if r := recover(); r != nil { glog.Error(c.Id(), " onMessage ", method, " recovered from panic: ", r) + debug.PrintStack() e := resultError{} e.Error.Message = "Internal error" rv = e @@ -214,7 +216,7 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res lower, higher := uint32(opts.End), uint32(opts.Start) for _, address := range addr { if !opts.QueryMempoolOnly { - err = s.db.GetTransactions(address, lower, higher, func(txid string, vout uint32, isOutput bool) error { + err = s.db.GetTransactions(address, lower, higher, func(txid string, vout int32, isOutput bool) error { txids = append(txids, txid) return nil }) @@ -222,11 +224,13 @@ func (s *SocketIoServer) getAddressTxids(addr []string, opts *addrOpts) (res res return res, err } } else { - m, err := s.chain.GetMempoolTransactions(address) + o, err := s.chain.GetMempoolTransactions(address) if err != nil { return res, err } - txids = append(txids, m...) + for _, m := range o { + txids = append(txids, m.Txid) + } } } res.Result = api.UniqueTxidsInReverse(txids) @@ -302,13 +306,13 @@ func txToResTx(tx *api.Tx) resTx { for i := range tx.Vin { vin := &tx.Vin[i] txid := vin.Txid - script := vin.ScriptSig.Hex + script := vin.Hex input := txInputs{ Txid: &txid, Script: &script, Sequence: int64(vin.Sequence), OutputIndex: int(vin.Vout), - Satoshis: vin.ValueSat.Int64(), + Satoshis: (*big.Int)(vin.ValueSat).Int64(), } if len(vin.Addresses) > 0 { a := vin.Addresses[0] @@ -319,13 +323,13 @@ func txToResTx(tx *api.Tx) resTx { outputs := make([]txOutputs, len(tx.Vout)) for i := range tx.Vout { vout := &tx.Vout[i] - script := vout.ScriptPubKey.Hex + script := vout.Hex output := txOutputs{ - Satoshis: vout.ValueSat.Int64(), + Satoshis: (*big.Int)(vout.ValueSat).Int64(), Script: &script, } - if len(vout.ScriptPubKey.Addresses) > 0 { - a := vout.ScriptPubKey.Addresses[0] + if len(vout.Addresses) > 0 { + a := vout.Addresses[0] output.Address = &a } outputs[i] = output @@ -338,15 +342,15 @@ func txToResTx(tx *api.Tx) resTx { } return resTx{ BlockTimestamp: tx.Blocktime, - FeeSatoshis: tx.FeesSat.Int64(), + FeeSatoshis: (*big.Int)(tx.FeesSat).Int64(), Hash: tx.Txid, Height: h, Hex: tx.Hex, Inputs: inputs, - InputSatoshis: tx.ValueInSat.Int64(), + InputSatoshis: (*big.Int)(tx.ValueInSat).Int64(), Locktime: int(tx.Locktime), Outputs: outputs, - OutputSatoshis: tx.ValueOutSat.Int64(), + OutputSatoshis: (*big.Int)(tx.ValueOutSat).Int64(), Version: int(tx.Version), } } @@ -387,7 +391,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r to = opts.To } for txi := opts.From; txi < to; txi++ { - tx, err := s.api.GetTransaction(txids[txi], false) + tx, err := s.api.GetTransaction(txids[txi], false, false) if err != nil { return res, err } @@ -403,12 +407,12 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r ads[a] = hi } hi.InputIndexes = append(hi.InputIndexes, int(vin.N)) - totalSat.Sub(&totalSat, &vin.ValueSat) + totalSat.Sub(&totalSat, (*big.Int)(vin.ValueSat)) } } for i := range tx.Vout { vout := &tx.Vout[i] - a := addressInSlice(vout.ScriptPubKey.Addresses, addr) + a := addressInSlice(vout.Addresses, addr) if a != "" { hi := ads[a] if hi == nil { @@ -416,7 +420,7 @@ func (s *SocketIoServer) getAddressHistory(addr []string, opts *addrOpts) (res r ads[a] = hi } hi.OutputIndexes = append(hi.OutputIndexes, int(vout.N)) - totalSat.Add(&totalSat, &vout.ValueSat) + totalSat.Add(&totalSat, (*big.Int)(vout.ValueSat)) } } ahi := addressHistoryItem{} @@ -579,10 +583,7 @@ type resultGetInfo struct { } func (s *SocketIoServer) getInfo() (res resultGetInfo, err error) { - height, _, err := s.db.GetBestBlock() - if err != nil { - return - } + _, height, _ := s.is.GetSyncState() res.Result.Blocks = int(height) res.Result.Testnet = s.chain.IsTestnet() res.Result.Network = s.chain.GetNetworkName() @@ -627,7 +628,7 @@ type resultGetDetailedTransaction struct { } func (s *SocketIoServer) getDetailedTransaction(txid string) (res resultGetDetailedTransaction, err error) { - tx, err := s.api.GetTransaction(txid, false) + tx, err := s.api.GetTransaction(txid, false, false) if err != nil { return res, err } @@ -664,6 +665,7 @@ func (s *SocketIoServer) onSubscribe(c *gosocketio.Channel, req []byte) interfac defer func() { if r := recover(); r != nil { glog.Error(c.Id(), " onSubscribe recovered from panic: ", r) + debug.PrintStack() } }() @@ -720,15 +722,12 @@ func (s *SocketIoServer) OnNewBlockHash(hash string) { } // OnNewTxAddr notifies users subscribed to bitcoind/addresstxid about new block -func (s *SocketIoServer) OnNewTxAddr(txid string, desc bchain.AddressDescriptor, isOutput bool) { +func (s *SocketIoServer) OnNewTxAddr(txid string, desc bchain.AddressDescriptor) { addr, searchable, err := s.chainParser.GetAddressesFromAddrDesc(desc) if err != nil { glog.Error("GetAddressesFromAddrDesc error ", err, " for descriptor ", desc) } else if searchable && len(addr) == 1 { data := map[string]interface{}{"address": addr[0], "txid": txid} - if !isOutput { - data["input"] = true - } c := s.server.BroadcastTo("bitcoind/addresstxid-"+string(desc), "bitcoind/addresstxid", data) if c > 0 { glog.Info("broadcasting new txid ", txid, " for addr ", addr[0], " to ", c, " channels") diff --git a/server/websocket.go b/server/websocket.go new file mode 100644 index 00000000..afda0c16 --- /dev/null +++ b/server/websocket.go @@ -0,0 +1,575 @@ +package server + +import ( + "blockbook/api" + "blockbook/bchain" + "blockbook/common" + "blockbook/db" + "encoding/json" + "math/big" + "net/http" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/golang/glog" + "github.com/gorilla/websocket" + "github.com/juju/errors" +) + +const upgradeFailed = "Upgrade failed: " +const outChannelSize = 500 +const defaultTimeout = 60 * time.Second + +var ( + // ErrorMethodNotAllowed is returned when client tries to upgrade method other than GET + ErrorMethodNotAllowed = errors.New("Method not allowed") + + connectionCounter uint64 +) + +type websocketReq struct { + ID string `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` +} + +type websocketRes struct { + ID string `json:"id"` + Data interface{} `json:"data"` +} + +type websocketChannel struct { + id uint64 + conn *websocket.Conn + out chan *websocketRes + ip string + requestHeader http.Header + alive bool + aliveLock sync.Mutex +} + +// WebsocketServer is a handle to websocket server +type WebsocketServer struct { + socket *websocket.Conn + upgrader *websocket.Upgrader + db *db.RocksDB + txCache *db.TxCache + chain bchain.BlockChain + chainParser bchain.BlockChainParser + metrics *common.Metrics + is *common.InternalState + api *api.Worker + block0hash string + newBlockSubscriptions map[*websocketChannel]string + newBlockSubscriptionsLock sync.Mutex + addressSubscriptions map[string]map[*websocketChannel]string + addressSubscriptionsLock sync.Mutex +} + +// NewWebsocketServer creates new websocket interface to blockbook and returns its handle +func NewWebsocketServer(db *db.RocksDB, chain bchain.BlockChain, txCache *db.TxCache, metrics *common.Metrics, is *common.InternalState) (*WebsocketServer, error) { + api, err := api.NewWorker(db, chain, txCache, is) + if err != nil { + return nil, err + } + b0, err := db.GetBlockHash(0) + if err != nil { + return nil, err + } + s := &WebsocketServer{ + upgrader: &websocket.Upgrader{ + ReadBufferSize: 1024 * 32, + WriteBufferSize: 1024 * 32, + CheckOrigin: checkOrigin, + }, + db: db, + txCache: txCache, + chain: chain, + chainParser: chain.GetChainParser(), + metrics: metrics, + is: is, + api: api, + block0hash: b0, + newBlockSubscriptions: make(map[*websocketChannel]string), + addressSubscriptions: make(map[string]map[*websocketChannel]string), + } + return s, nil +} + +// allow all origins +func checkOrigin(r *http.Request) bool { + return true +} + +// ServeHTTP sets up handler of websocket channel +func (s *WebsocketServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + http.Error(w, upgradeFailed+ErrorMethodNotAllowed.Error(), 503) + return + } + conn, err := s.upgrader.Upgrade(w, r, nil) + if err != nil { + http.Error(w, upgradeFailed+err.Error(), 503) + return + } + c := &websocketChannel{ + id: atomic.AddUint64(&connectionCounter, 1), + conn: conn, + out: make(chan *websocketRes, outChannelSize), + ip: r.RemoteAddr, + requestHeader: r.Header, + alive: true, + } + go s.inputLoop(c) + go s.outputLoop(c) + s.onConnect(c) +} + +// GetHandler returns http handler +func (s *WebsocketServer) GetHandler() http.Handler { + return s +} + +func (s *WebsocketServer) closeChannel(c *websocketChannel) { + c.aliveLock.Lock() + defer c.aliveLock.Unlock() + if c.alive { + c.conn.Close() + c.alive = false + //clean out + close(c.out) + for len(c.out) > 0 { + <-c.out + } + s.onDisconnect(c) + } +} + +func (c *websocketChannel) IsAlive() bool { + c.aliveLock.Lock() + defer c.aliveLock.Unlock() + return c.alive +} + +func (s *WebsocketServer) inputLoop(c *websocketChannel) { + defer func() { + if r := recover(); r != nil { + glog.Error("recovered from panic: ", r, ", ", c.id) + debug.PrintStack() + s.closeChannel(c) + } + }() + for { + t, d, err := c.conn.ReadMessage() + if err != nil { + s.closeChannel(c) + return + } + switch t { + case websocket.TextMessage: + var req websocketReq + err := json.Unmarshal(d, &req) + if err != nil { + glog.Error("Error parsing message from ", c.id, ", ", string(d), ", ", err) + s.closeChannel(c) + return + } + go s.onRequest(c, &req) + case websocket.BinaryMessage: + glog.Error("Binary message received from ", c.id, ", ", c.ip) + s.closeChannel(c) + return + case websocket.PingMessage: + c.conn.WriteControl(websocket.PongMessage, nil, time.Now().Add(defaultTimeout)) + break + case websocket.CloseMessage: + s.closeChannel(c) + return + case websocket.PongMessage: + // do nothing + } + } +} + +func (s *WebsocketServer) outputLoop(c *websocketChannel) { + for m := range c.out { + err := c.conn.WriteJSON(m) + if err != nil { + glog.Error("Error sending message to ", c.id, ", ", err) + s.closeChannel(c) + } + } +} + +func (s *WebsocketServer) onConnect(c *websocketChannel) { + glog.Info("Client connected ", c.id, ", ", c.ip) + s.metrics.WebsocketClients.Inc() +} + +func (s *WebsocketServer) onDisconnect(c *websocketChannel) { + s.unsubscribeNewBlock(c) + s.unsubscribeAddresses(c) + glog.Info("Client disconnected ", c.id, ", ", c.ip) + s.metrics.WebsocketClients.Dec() +} + +var requestHandlers = map[string]func(*WebsocketServer, *websocketChannel, *websocketReq) (interface{}, error){ + "getAccountInfo": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + r, err := unmarshalGetAccountInfoRequest(req.Params) + if err == nil { + rv, err = s.getAccountInfo(r) + } + return + }, + "getInfo": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + return s.getInfo() + }, + "estimateFee": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + return s.estimateFee(c, req.Params) + }, + "sendTransaction": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + r := struct { + Hex string `json:"hex"` + }{} + err = json.Unmarshal(req.Params, &r) + if err == nil { + rv, err = s.sendTransaction(r.Hex) + } + return + }, + "subscribeNewBlock": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + return s.subscribeNewBlock(c, req) + }, + "unsubscribeNewBlock": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + return s.unsubscribeNewBlock(c) + }, + "subscribeAddresses": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + ad, err := s.unmarshalAddresses(req.Params) + if err == nil { + rv, err = s.subscribeAddresses(c, ad, req) + } + return + }, + "unsubscribeAddresses": func(s *WebsocketServer, c *websocketChannel, req *websocketReq) (rv interface{}, err error) { + return s.unsubscribeAddresses(c) + }, +} + +func (s *WebsocketServer) onRequest(c *websocketChannel, req *websocketReq) { + var err error + var data interface{} + defer func() { + if r := recover(); r != nil { + glog.Error("Client ", c.id, ", onRequest ", req.Method, " recovered from panic: ", r) + debug.PrintStack() + e := resultError{} + e.Error.Message = "Internal error" + data = e + } + // nil data means no response + if data != nil { + c.out <- &websocketRes{ + ID: req.ID, + Data: data, + } + } + }() + t := time.Now() + defer s.metrics.WebsocketReqDuration.With(common.Labels{"method": req.Method}).Observe(float64(time.Since(t)) / 1e3) // in microseconds + f, ok := requestHandlers[req.Method] + if ok { + data, err = f(s, c, req) + } else { + err = errors.New("unknown method") + } + if err == nil { + glog.V(1).Info("Client ", c.id, " onRequest ", req.Method, " success") + s.metrics.SocketIORequests.With(common.Labels{"method": req.Method, "status": "success"}).Inc() + } else { + glog.Error("Client ", c.id, " onMessage ", req.Method, ": ", errors.ErrorStack(err)) + s.metrics.SocketIORequests.With(common.Labels{"method": req.Method, "status": err.Error()}).Inc() + e := resultError{} + e.Error.Message = err.Error() + data = e + } +} + +type accountInfoReq struct { + Descriptor string `json:"descriptor"` + Details string `json:"details"` + PageSize int `json:"pageSize"` + Page int `json:"page"` + FromHeight int `json:"from"` + ToHeight int `json:"to"` + ContractFilter string `json:"contractFilter"` +} + +func unmarshalGetAccountInfoRequest(params []byte) (*accountInfoReq, error) { + var r accountInfoReq + err := json.Unmarshal(params, &r) + if err != nil { + return nil, err + } + return &r, nil +} + +func (s *WebsocketServer) getAccountInfo(req *accountInfoReq) (res *api.Address, err error) { + var opt api.GetAddressOption + switch req.Details { + case "balance": + opt = api.Balance + case "txids": + opt = api.TxidHistory + case "txs": + opt = api.TxHistory + default: + opt = api.Basic + } + + return s.api.GetAddress(req.Descriptor, req.Page, req.PageSize, opt, &api.AddressFilter{ + FromHeight: uint32(req.FromHeight), + ToHeight: uint32(req.ToHeight), + Contract: req.ContractFilter, + Vout: api.AddressFilterVoutOff, + }) +} + +func (s *WebsocketServer) getInfo() (interface{}, error) { + vi := common.GetVersionInfo() + height, hash, err := s.db.GetBestBlock() + if err != nil { + return nil, err + } + type info struct { + Name string `json:"name"` + Shortcut string `json:"shortcut"` + Decimals int `json:"decimals"` + Version string `json:"version"` + BestHeight int `json:"bestheight"` + BestHash string `json:"besthash"` + Block0Hash string `json:"block0hash"` + Testnet bool `json:"testnet"` + } + return &info{ + Name: s.is.Coin, + Shortcut: s.is.CoinShortcut, + Decimals: s.chainParser.AmountDecimals(), + BestHeight: int(height), + BestHash: hash, + Version: vi.Version, + Block0Hash: s.block0hash, + Testnet: s.chain.IsTestnet(), + }, nil +} + +func (s *WebsocketServer) estimateFee(c *websocketChannel, params []byte) (interface{}, error) { + type estimateFeeReq struct { + Blocks []int `json:"blocks"` + Specific map[string]interface{} `json:"specific"` + } + type estimateFeeRes struct { + FeePerTx string `json:"feePerTx,omitempty"` + FeePerUnit string `json:"feePerUnit,omitempty"` + FeeLimit string `json:"feeLimit,omitempty"` + } + var r estimateFeeReq + err := json.Unmarshal(params, &r) + if err != nil { + return nil, err + } + res := make([]estimateFeeRes, len(r.Blocks)) + if s.chainParser.GetChainType() == bchain.ChainEthereumType { + gas, err := s.chain.EthereumTypeEstimateGas(r.Specific) + if err != nil { + return nil, err + } + sg := strconv.FormatUint(gas, 10) + for i, b := range r.Blocks { + fee, err := s.chain.EstimateSmartFee(b, true) + if err != nil { + return nil, err + } + res[i].FeePerUnit = fee.String() + res[i].FeeLimit = sg + fee.Mul(&fee, new(big.Int).SetUint64(gas)) + res[i].FeePerTx = fee.String() + } + } else { + conservative := true + v, ok := r.Specific["conservative"] + if ok { + vc, ok := v.(bool) + if ok { + conservative = vc + } + } + txSize := 0 + v, ok = r.Specific["txsize"] + if ok { + f, ok := v.(float64) + if ok { + txSize = int(f) + } + } + for i, b := range r.Blocks { + fee, err := s.chain.EstimateSmartFee(b, conservative) + if err != nil { + return nil, err + } + res[i].FeePerUnit = fee.String() + if txSize > 0 { + fee.Mul(&fee, big.NewInt(int64(txSize))) + fee.Add(&fee, big.NewInt(500)) + fee.Div(&fee, big.NewInt(1000)) + res[i].FeePerTx = fee.String() + } + } + } + return res, nil +} + +func (s *WebsocketServer) sendTransaction(tx string) (res resultSendTransaction, err error) { + txid, err := s.chain.SendRawTransaction(tx) + if err != nil { + return res, err + } + res.Result = txid + return +} + +type subscriptionResponse struct { + Subscribed bool `json:"subscribed"` +} + +func (s *WebsocketServer) subscribeNewBlock(c *websocketChannel, req *websocketReq) (res interface{}, err error) { + s.newBlockSubscriptionsLock.Lock() + defer s.newBlockSubscriptionsLock.Unlock() + s.newBlockSubscriptions[c] = req.ID + return &subscriptionResponse{true}, nil +} + +func (s *WebsocketServer) unsubscribeNewBlock(c *websocketChannel) (res interface{}, err error) { + s.newBlockSubscriptionsLock.Lock() + defer s.newBlockSubscriptionsLock.Unlock() + delete(s.newBlockSubscriptions, c) + return &subscriptionResponse{false}, nil +} + +func (s *WebsocketServer) unmarshalAddresses(params []byte) ([]bchain.AddressDescriptor, error) { + r := struct { + Addresses []string `json:"addresses"` + }{} + err := json.Unmarshal(params, &r) + if err != nil { + return nil, err + } + rv := make([]bchain.AddressDescriptor, len(r.Addresses)) + for i, a := range r.Addresses { + ad, err := s.chainParser.GetAddrDescFromAddress(a) + if err != nil { + return nil, err + } + rv[i] = ad + } + return rv, nil +} + +func (s *WebsocketServer) subscribeAddresses(c *websocketChannel, addrDesc []bchain.AddressDescriptor, req *websocketReq) (res interface{}, err error) { + // unsubscribe all previous subscriptions + s.unsubscribeAddresses(c) + s.addressSubscriptionsLock.Lock() + defer s.addressSubscriptionsLock.Unlock() + for i := range addrDesc { + ads := string(addrDesc[i]) + as, ok := s.addressSubscriptions[ads] + if !ok { + as = make(map[*websocketChannel]string) + s.addressSubscriptions[ads] = as + } + as[c] = req.ID + } + return &subscriptionResponse{true}, nil +} + +// unsubscribeAddresses unsubscribes all address subscriptions by this channel +func (s *WebsocketServer) unsubscribeAddresses(c *websocketChannel) (res interface{}, err error) { + s.addressSubscriptionsLock.Lock() + defer s.addressSubscriptionsLock.Unlock() + for _, sa := range s.addressSubscriptions { + for sc := range sa { + if sc == c { + delete(sa, c) + } + } + } + return &subscriptionResponse{false}, nil +} + +// OnNewBlock is a callback that broadcasts info about new block to subscribed clients +func (s *WebsocketServer) OnNewBlock(hash string, height uint32) { + s.newBlockSubscriptionsLock.Lock() + defer s.newBlockSubscriptionsLock.Unlock() + data := struct { + Height uint32 `json:"height"` + Hash string `json:"hash"` + }{ + Height: height, + Hash: hash, + } + for c, id := range s.newBlockSubscriptions { + if c.IsAlive() { + c.out <- &websocketRes{ + ID: id, + Data: &data, + } + } + } + glog.Info("broadcasting new block ", height, " ", hash, " to ", len(s.newBlockSubscriptions), " channels") +} + +// OnNewTxAddr is a callback that broadcasts info about a tx affecting subscribed address +func (s *WebsocketServer) OnNewTxAddr(tx *bchain.Tx, addrDesc bchain.AddressDescriptor) { + // check if there is any subscription but release the lock immediately, GetTransactionFromBchainTx may take some time + s.addressSubscriptionsLock.Lock() + as, ok := s.addressSubscriptions[string(addrDesc)] + s.addressSubscriptionsLock.Unlock() + if ok && len(as) > 0 { + addr, _, err := s.chainParser.GetAddressesFromAddrDesc(addrDesc) + if err != nil { + glog.Error("GetAddressesFromAddrDesc error ", err, " for ", addrDesc) + return + } + if len(addr) == 1 { + atx, err := s.api.GetTransactionFromBchainTx(tx, 0, false, false) + if err != nil { + glog.Error("GetTransactionFromBchainTx error ", err, " for ", tx.Txid) + return + } + data := struct { + Address string `json:"address"` + Tx *api.Tx `json:"tx"` + }{ + Address: addr[0], + Tx: atx, + } + // get the list of subscriptions again, this time keep the lock + s.addressSubscriptionsLock.Lock() + defer s.addressSubscriptionsLock.Unlock() + as, ok = s.addressSubscriptions[string(addrDesc)] + if ok { + for c, id := range as { + if c.IsAlive() { + c.out <- &websocketRes{ + ID: id, + Data: &data, + } + } + } + glog.Info("broadcasting new tx ", tx.Txid, " for addr ", addr[0], " to ", len(as), " channels") + } + } + } +} diff --git a/static/css/main.css b/static/css/main.css index 28e7344e..c4ef6a48 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -214,6 +214,11 @@ h3 { font-weight: bold; } +table.data-table table.data-table th { + border-top: 0; + font-weight: normal; +} + .alert .data-table { margin: 0; } @@ -239,19 +244,7 @@ h3 { font-size: 14px; } -.h-container { - display: -webkit-box; - display: -ms-flexbox; - -ms-flex-wrap: wrap; - flex-wrap: wrap; -} - -.h-container-6 { - width: 50%; - margin: 0; -} - -.h-container ul { +.h-container ul, .h-container h3 { margin: 0; } diff --git a/static/templates/address.html b/static/templates/address.html index 7a56e729..508929fe 100644 --- a/static/templates/address.html +++ b/static/templates/address.html @@ -1,6 +1,6 @@ {{define "specific"}}{{$cs := .CoinShortcut}}{{$addr := .Address}}{{$data := .}} -

Address - {{formatAmount $addr.Balance}} {{$cs}} +

{{if $addr.Erc20Contract}}Contract {{$addr.Erc20Contract.Name}} ({{$addr.Erc20Contract.Symbol}}){{else}}Address{{end}} + {{formatAmount $addr.BalanceSat}} {{$cs}}

{{$addr.AddrStr}} @@ -10,22 +10,61 @@
+ {{- if eq .ChainType 1 -}} + + + + + + + + + + + + + {{- if $addr.Erc20Tokens -}} + + + + + {{- end -}} + + {{- else -}} - + - + - + + {{- end -}}
Balance{{formatAmount $addr.BalanceSat}} {{$cs}}
Non-contract Transactions{{$addr.TxApperances}}
Nonce{{$addr.Nonce}}
ERC20 Tokens + + + + + + + + {{- range $et := $addr.Erc20Tokens -}} + + + + + + {{- end -}} + +
ContractTokensTransfers
{{$et.Name}}{{formatAmountWithDecimals $et.BalanceSat $et.Decimals}} {{$et.Symbol}}{{$et.Transfers}}
+
Total Received{{formatAmount $addr.TotalReceived}} {{$cs}}{{formatAmount $addr.TotalReceivedSat}} {{$cs}}
Total Sent{{formatAmount $addr.TotalSent}} {{$cs}}{{formatAmount $addr.TotalSentSat}} {{$cs}}
Final Balance{{formatAmount $addr.Balance}} {{$cs}}{{formatAmount $addr.BalanceSat}} {{$cs}}
No. Transactions {{$addr.TxApperances}}
@@ -44,7 +83,7 @@ Unconfirmed Balance - {{formatAmount $addr.UnconfirmedBalance}} {{$cs}} + {{formatAmount $addr.UnconfirmedBalanceSat}} {{$cs}} No. Transactions @@ -53,10 +92,23 @@
-{{- end}}{{if $addr.Transactions -}} -
-

Transactions

- +{{- end}}{{if or $addr.Transactions $addr.Filter -}} +
+

Transactions

+ +
+ +
{{- range $tx := $addr.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}} diff --git a/static/templates/block.html b/static/templates/block.html index fc896425..87d4f2f8 100644 --- a/static/templates/block.html +++ b/static/templates/block.html @@ -3,9 +3,9 @@
{{$b.Hash}}
-
-

Summary

-
{{- if $b.Transactions -}} -
-

Transactions

- +
+

Transactions

+
{{- range $tx := $b.Transactions}}{{$data := setTxToTemplateData $data $tx}}{{template "txdetail" $data}}{{end -}} diff --git a/static/templates/index.html b/static/templates/index.html index bf37c544..f6638adc 100644 --- a/static/templates/index.html +++ b/static/templates/index.html @@ -78,10 +78,12 @@ Difficulty {{$be.Difficulty}} + {{- if $be.Timeoffset -}} Timeoffset {{$be.Timeoffset}} + {{- end -}} {{- if $be.SizeOnDisk -}} Size On Disk diff --git a/static/templates/paging.html b/static/templates/paging.html index fa9e7245..c8260779 100644 --- a/static/templates/paging.html +++ b/static/templates/paging.html @@ -1,11 +1,11 @@ {{- define "paging"}}{{$data := . -}}{{if $data.PagingRange -}}
    -
  • <
    • +
  • <
    • {{- range $p := $data.PagingRange -}}
  • - {{- if $p}}{{$p}} + {{- if $p}}{{$p}} {{- else -}}...{{- end -}}
    • {{- end -}} -
  • >
    • +
  • >
{{- end -}}{{- end -}} \ No newline at end of file diff --git a/static/templates/tx.html b/static/templates/tx.html index b04983ba..ee4279f1 100644 --- a/static/templates/tx.html +++ b/static/templates/tx.html @@ -1,4 +1,4 @@ -{{define "specific"}}{{$cs := .CoinShortcut}}{{$tx := .Tx}}{{$txSpecific := .TxSpecific}} +{{define "specific"}}{{$cs := .CoinShortcut}}{{$tx := .Tx}}

Transaction

{{$tx.Txid}} @@ -7,33 +7,64 @@
- {{if $tx.Confirmations}} + {{- if $tx.Confirmations -}} - {{end}} + {{end -}} - {{if $tx.Confirmations}} + {{- if $tx.Confirmations -}} {{end}} + {{- if $tx.EthereumSpecific -}} + + + {{- if $tx.EthereumSpecific.Status -}} + {{- if eq $tx.EthereumSpecific.Status 1 -}} + + {{- else -}} + {{- if eq $tx.EthereumSpecific.Status -1 -}} + + {{- else -}} + + {{- end -}} + {{- end -}} + {{- else -}} + + {{- end -}} + + + + + + + + + + + + + + {{- else -}} - + - + - {{if $tx.Fees}} + {{- end -}} + {{- if $tx.FeesSat -}} - - {{end}} + + {{end -}}
Mined Time {{formatUnixTime $tx.Blocktime}}
In Block {{if $tx.Confirmations}}{{$tx.Blockhash}}{{else}}Unconfirmed{{end}}
In Block Height {{$tx.Blockheight}}
StatusSuccessPendingUnknownFail
Value{{formatAmount $tx.ValueOutSat}} {{$cs}}
Gas Used / Limit{{if $tx.EthereumSpecific.GasUsed}}{{$tx.EthereumSpecific.GasUsed}}{{else}}pending{{end}} / {{$tx.EthereumSpecific.GasLimit}}
Gas Price{{formatAmount $tx.EthereumSpecific.GasPrice}} {{$cs}}
Total Input{{formatAmount $tx.ValueIn}} {{$cs}}{{formatAmount $tx.ValueInSat}} {{$cs}}
Total Output{{formatAmount $tx.ValueOut}} {{$cs}}{{formatAmount $tx.ValueOutSat}} {{$cs}}
Fees{{formatAmount $tx.Fees}} {{$cs}}
{{formatAmount $tx.FeesSat}} {{$cs}}
@@ -47,7 +78,7 @@ 

     

     
-    Test socket.io
+    Blockbook Socket.io Test Page
     
+
+
+
+    

+        

+            
Blockbook Websocket Test Page

+        

+        

+            

+                
+            

+            

+                
+            

+            

+                
+            

+        

+        

+            

+                
+            

+            

+            

+        

+        

+            

+                
+            

+            

+                

+                    
+                    
+                

+                

+                    
+                    
+                    
+                    
+                

+            

+            

+        

+        

+            

+            

+        

+        

+            

+                
+            

+            

+                

+                    
+                

+                

+                    
+                

+            

+            

+        

+        

+            

+            

+        

+        

+            

+                
+            

+            

+                
+            

+            

+            

+        

+        

+            

+        

+        

+            

+                
+            

+            

+                
+            

+            

+                
+            

+        

+        

+            

+        

+        

+            

+                
+            

+            

+                
+            

+            

+                
+            

+            

+                
+            

+        

+        

+            

+        

+    

+
+
+
+
\ No newline at end of file
diff --git a/tests/dbtestdata/dbtestdata.go b/tests/dbtestdata/dbtestdata.go
index 1c4ad328..c485129f 100644
--- a/tests/dbtestdata/dbtestdata.go
+++ b/tests/dbtestdata/dbtestdata.go
@@ -54,7 +54,7 @@ func AddressToPubKeyHex(addr string, parser bchain.BlockChainParser) string {
 	return hex.EncodeToString(b)
 }
 
-func GetTestUTXOBlock1(parser bchain.BlockChainParser) *bchain.Block {
+func GetTestBitcoinTypeBlock1(parser bchain.BlockChainParser) *bchain.Block {
 	return &bchain.Block{
 		BlockHeader: bchain.BlockHeader{
 			Height:        225493,
@@ -66,6 +66,7 @@ func GetTestUTXOBlock1(parser bchain.BlockChainParser) *bchain.Block {
 		Txs: []bchain.Tx{
 			bchain.Tx{
 				Txid: TxidB1T1,
+				Vin:  []bchain.Vin{},
 				Vout: []bchain.Vout{
 					bchain.Vout{
 						N: 0,
@@ -119,7 +120,7 @@ func GetTestUTXOBlock1(parser bchain.BlockChainParser) *bchain.Block {
 	}
 }
 
-func GetTestUTXOBlock2(parser bchain.BlockChainParser) *bchain.Block {
+func GetTestBitcoinTypeBlock2(parser bchain.BlockChainParser) *bchain.Block {
 	return &bchain.Block{
 		BlockHeader: bchain.BlockHeader{
 			Height:        225494,
diff --git a/tests/dbtestdata/dbtestdata_ethereumtype.go b/tests/dbtestdata/dbtestdata_ethereumtype.go
new file mode 100644
index 00000000..04ca9956
--- /dev/null
+++ b/tests/dbtestdata/dbtestdata_ethereumtype.go
@@ -0,0 +1,69 @@
+package dbtestdata
+
+import (
+	"blockbook/bchain"
+	"encoding/hex"
+)
+
+const (
+	EthAddr3e         = "3e3a3d69dc66ba10737f531ed088954a9ec89d97"
+	EthAddr55         = "555ee11fbddc0e49a9bab358a8941ad95ffdb48f"
+	EthAddr20         = "20cd153de35d469ba46127a0c8f18626b59a256a"
+	EthAddr9f         = "9f4981531fda132e83c44680787dfa7ee31e4f8d"
+	EthAddr4b         = "4bda106325c335df99eab7fe363cac8a0ba2a24d"
+	EthAddr7b         = "7b62eb7fe80350dc7ec945c0b73242cb9877fb1b"
+	EthAddrContract4a = "4af4114f73d1c1c903ac9e0361b379d1291808a2" // ERC-20 (VTY)
+	EthAddrContract0d = "0d0f936ee4c93e25944694d6c121de94d9760f11" // ERC-20 (MTT)
+	EthAddrContract47 = "479cc461fecd078f766ecc58533d6f69580cf3ac" // non ERC20
+
+	EthTxidB1T1  = "cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b"
+	EthTx1Packed = "08e8dd870210a6a6f0db051a6908ece40212050430e234001888a40122081bc0159d530e60003220cd647151552b5132b2aef7c9be00dc6f73afc5901dde157aab131335baaa853b3a14555ee11fbddc0e49a9bab358a8941ad95ffdb48f42143e3a3d69dc66ba10737f531ed088954a9ec89d97480a22070a025208120101"
+	EthTxidB1T2  = "a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b101"
+	EthTx2Packed = "08e8dd870210a6a6f0db051aa20108d001120509502f900018d5e1042a44a9059cbb000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f00000000000000000000000000000000000000000000021e19e0c9bab24000003220a9cd088aba2131000da6f38a33c20169baee476218deea6b78720700b895b1013a144af4114f73d1c1c903ac9e0361b379d1291808a2421420cd153de35d469ba46127a0c8f18626b59a256a22a8010a02cb391201011a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000021e19e0c9bab24000001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a2000000000000000000000000020cd153de35d469ba46127a0c8f18626b59a256a1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f"
+	EthTxidB2T1  = "c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c91"
+	EthTx3Packed = "08e9dd870210d4b5f0db051a6708c20112050218711a001888a401220710bc3578bd37d83220c2c3dd1ecb00e8a6d81f793d24387cf2947a313e94ab03b1fb22cd63320f6c913a149f4981531fda132e83c44680787dfa7ee31e4f8d4214555ee11fbddc0e49a9bab358a8941ad95ffdb48f480722070a025208120101"
+	EthTxidB2T2  = "c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf2"
+	EthTx4Packed = "08e9dd870210d4b5f0db051aa50b08f6be0712043b9aca001890a10f2ac40a4f15078700000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000000000022000000000000000000000000000000000000000000000000000000000000003c00000000000000000000000000000000000000000000000000000000000000420000000000000000000000000000000000000000000000000000000000000048000000000000000000000000000000000000000000000000000000000000004e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a200000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000a5ef5a7656bfb0000000000000000000000000000000000000000000000000000004ba78398d5c5000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfe0b9579b4ecf7a2801880f644009a324671a79754ea57c3a103c6e70d3dbef6ba69a08000000000000000000000000000000000000000000000000004f937d86afb90000000000000000000000000000000000000000000000000ab280fd8037d500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfb784b7c1f3fbe8b75484603ab8adc58aaee3a46245a6579fac7077b5570018b4e0d4eb0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000308fd0e798ac00000000000000000000000000000000000000000000000006a8313d60b1f80000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000001b000000000000000000000000000000000000000000000000000000000000001b00000000000000000000000000000000000000000000000000000000000000029de0ccec59e8948e3d905b40e5542335ebc1eb4674db517d2f6392ec7fdeb3d45f3449d313ee2589819c6c79eb1c1b047adae68565c1608e3a1d1d70823febb0000000000000000000000000000000000000000000000000000000000000000234d06fe17f1202e8b07177a30eb64d14adc08cdb3fa1b3e3e0bea0f9672c02175b77c01c51d3c7e460723b27ecbc7801fd6482559a8c9999593f9a4d149c73843220c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf23a14479cc461fecd078f766ecc58533d6f69580cf3ac42144bda106325c335df99eab7fe363cac8a0ba2a24d482422d40b0a03034d301201011a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f606b1a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a21220000000000000000000000000000000000000000000000000000308fd0e798ac01a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000000000000000000000000006a8313d60b1f606b000000000000000000000000000000000000000000000000000308fd0e798ac0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005e083a16f4b092c5729a49f9c3ed3cc171bb3d3d0c22e20b1de6063c32f399ac1a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a2000000000000000000000000000000000000000000000000000000000000000001a205af266c0a89a07c1917deaa024414577e6c3c31c8907d079e13eb448c082594f1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000000000031855667df7a81a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f80001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f2b0d62c44ed08f2a5adef40c875d20310a42a9d4f488bd26323256fe01c7f481a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a2000000000000000000000000000000000000000000000000000000000000000001a20b0b69dad58df6032c3b266e19b1045b19c87acd2c06fb0c598090f44b8e263aa"
+)
+
+func unpackTxs(packed []string, parser bchain.BlockChainParser) []bchain.Tx {
+	r := make([]bchain.Tx, len(packed))
+	for i, p := range packed {
+		b, err := hex.DecodeString(p)
+		if err != nil {
+			panic(err)
+		}
+		tx, _, err := parser.UnpackTx(b)
+		if err != nil {
+			panic(err)
+		}
+		r[i] = *tx
+	}
+	return r
+}
+
+func GetTestEthereumTypeBlock1(parser bchain.BlockChainParser) *bchain.Block {
+	return &bchain.Block{
+		BlockHeader: bchain.BlockHeader{
+			Height:        4321000,
+			Hash:          "0xc7b98df95acfd11c51ba25611a39e004fe56c8fdfc1582af99354fcd09c17b11",
+			Size:          31839,
+			Time:          1534858022,
+			Confirmations: 2,
+		},
+		Txs: unpackTxs([]string{EthTx1Packed, EthTx2Packed}, parser),
+	}
+}
+
+func GetTestEthereumTypeBlock2(parser bchain.BlockChainParser) *bchain.Block {
+	return &bchain.Block{
+		BlockHeader: bchain.BlockHeader{
+			Height:        4321001,
+			Hash:          "0x2b57e15e93a0ed197417a34c2498b7187df79099572c04a6b6e6ff418f74e6ee",
+			Size:          2345678,
+			Time:          1534859988,
+			Confirmations: 1,
+		},
+		Txs: unpackTxs([]string{EthTx3Packed, EthTx4Packed}, parser),
+	}
+}
diff --git a/tests/dbtestdata/fakechain.go b/tests/dbtestdata/fakechain.go
index d29740b9..ffc0b2ce 100644
--- a/tests/dbtestdata/fakechain.go
+++ b/tests/dbtestdata/fakechain.go
@@ -9,11 +9,11 @@ import (
 )
 
 type fakeBlockChain struct {
-	parser bchain.BlockChainParser
+	*bchain.BaseChain
 }
 
 func NewFakeBlockChain(parser bchain.BlockChainParser) (*fakeBlockChain, error) {
-	return &fakeBlockChain{parser: parser}, nil
+	return &fakeBlockChain{&bchain.BaseChain{Parser: parser}}, nil
 }
 
 func (c *fakeBlockChain) Initialize() error {
@@ -45,26 +45,26 @@ func (c *fakeBlockChain) GetChainInfo() (v *bchain.ChainInfo, err error) {
 		Chain:         c.GetNetworkName(),
 		Blocks:        2,
 		Headers:       2,
-		Bestblockhash: GetTestUTXOBlock2(c.parser).BlockHeader.Hash,
+		Bestblockhash: GetTestBitcoinTypeBlock2(c.Parser).BlockHeader.Hash,
 		Version:       "001001",
 		Subversion:    c.GetSubversion(),
 	}, nil
 }
 
 func (c *fakeBlockChain) GetBestBlockHash() (v string, err error) {
-	return GetTestUTXOBlock2(c.parser).BlockHeader.Hash, nil
+	return GetTestBitcoinTypeBlock2(c.Parser).BlockHeader.Hash, nil
 }
 
 func (c *fakeBlockChain) GetBestBlockHeight() (v uint32, err error) {
-	return GetTestUTXOBlock2(c.parser).BlockHeader.Height, nil
+	return GetTestBitcoinTypeBlock2(c.Parser).BlockHeader.Height, nil
 }
 
 func (c *fakeBlockChain) GetBlockHash(height uint32) (v string, err error) {
-	b1 := GetTestUTXOBlock1(c.parser)
+	b1 := GetTestBitcoinTypeBlock1(c.Parser)
 	if height == b1.BlockHeader.Height {
 		return b1.BlockHeader.Hash, nil
 	}
-	b2 := GetTestUTXOBlock2(c.parser)
+	b2 := GetTestBitcoinTypeBlock2(c.Parser)
 	if height == b2.BlockHeader.Height {
 		return b2.BlockHeader.Hash, nil
 	}
@@ -72,11 +72,11 @@ func (c *fakeBlockChain) GetBlockHash(height uint32) (v string, err error) {
 }
 
 func (c *fakeBlockChain) GetBlockHeader(hash string) (v *bchain.BlockHeader, err error) {
-	b1 := GetTestUTXOBlock1(c.parser)
+	b1 := GetTestBitcoinTypeBlock1(c.Parser)
 	if hash == b1.BlockHeader.Hash {
 		return &b1.BlockHeader, nil
 	}
-	b2 := GetTestUTXOBlock2(c.parser)
+	b2 := GetTestBitcoinTypeBlock2(c.Parser)
 	if hash == b2.BlockHeader.Hash {
 		return &b2.BlockHeader, nil
 	}
@@ -84,11 +84,11 @@ func (c *fakeBlockChain) GetBlockHeader(hash string) (v *bchain.BlockHeader, err
 }
 
 func (c *fakeBlockChain) GetBlock(hash string, height uint32) (v *bchain.Block, err error) {
-	b1 := GetTestUTXOBlock1(c.parser)
+	b1 := GetTestBitcoinTypeBlock1(c.Parser)
 	if hash == b1.BlockHeader.Hash || height == b1.BlockHeader.Height {
 		return b1, nil
 	}
-	b2 := GetTestUTXOBlock2(c.parser)
+	b2 := GetTestBitcoinTypeBlock2(c.Parser)
 	if hash == b2.BlockHeader.Hash || height == b2.BlockHeader.Height {
 		return b2, nil
 	}
@@ -106,11 +106,11 @@ func getBlockInfo(b *bchain.Block) *bchain.BlockInfo {
 }
 
 func (c *fakeBlockChain) GetBlockInfo(hash string) (v *bchain.BlockInfo, err error) {
-	b1 := GetTestUTXOBlock1(c.parser)
+	b1 := GetTestBitcoinTypeBlock1(c.Parser)
 	if hash == b1.BlockHeader.Hash {
 		return getBlockInfo(b1), nil
 	}
-	b2 := GetTestUTXOBlock2(c.parser)
+	b2 := GetTestBitcoinTypeBlock2(c.Parser)
 	if hash == b2.BlockHeader.Hash {
 		return getBlockInfo(b2), nil
 	}
@@ -131,9 +131,9 @@ func getTxInBlock(b *bchain.Block, txid string) *bchain.Tx {
 }
 
 func (c *fakeBlockChain) GetTransaction(txid string) (v *bchain.Tx, err error) {
-	v = getTxInBlock(GetTestUTXOBlock1(c.parser), txid)
+	v = getTxInBlock(GetTestBitcoinTypeBlock1(c.Parser), txid)
 	if v == nil {
-		v = getTxInBlock(GetTestUTXOBlock2(c.parser), txid)
+		v = getTxInBlock(GetTestBitcoinTypeBlock2(c.Parser), txid)
 	}
 	if v != nil {
 		return v, nil
@@ -141,8 +141,8 @@ func (c *fakeBlockChain) GetTransaction(txid string) (v *bchain.Tx, err error) {
 	return nil, errors.New("Not found")
 }
 
-func (c *fakeBlockChain) GetTransactionSpecific(txid string) (v json.RawMessage, err error) {
-	tx, err := c.GetTransaction(txid)
+func (c *fakeBlockChain) GetTransactionSpecific(tx *bchain.Tx) (v json.RawMessage, err error) {
+	tx, err = c.GetTransaction(tx.Txid)
 	if err != nil {
 		return nil, err
 	}
@@ -182,12 +182,12 @@ func (c *fakeBlockChain) ResyncMempool(onNewTxAddr bchain.OnNewTxAddrFunc) (coun
 	return 0, errors.New("Not implemented")
 }
 
-func (c *fakeBlockChain) GetMempoolTransactions(address string) (v []string, err error) {
+func (c *fakeBlockChain) GetMempoolTransactions(address string) (v []bchain.Outpoint, err error) {
 	return nil, errors.New("Not implemented")
 }
 
-func (c *fakeBlockChain) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []string, err error) {
-	return []string{}, nil
+func (c *fakeBlockChain) GetMempoolTransactionsForAddrDesc(addrDesc bchain.AddressDescriptor) (v []bchain.Outpoint, err error) {
+	return []bchain.Outpoint{}, nil
 }
 
 func (c *fakeBlockChain) GetMempoolEntry(txid string) (v *bchain.MempoolEntry, err error) {
@@ -195,5 +195,5 @@ func (c *fakeBlockChain) GetMempoolEntry(txid string) (v *bchain.MempoolEntry, e
 }
 
 func (c *fakeBlockChain) GetChainParser() bchain.BlockChainParser {
-	return c.parser
+	return c.Parser
 }
diff --git a/tests/rpc/rpc.go b/tests/rpc/rpc.go
index b5f2216d..ff430d78 100644
--- a/tests/rpc/rpc.go
+++ b/tests/rpc/rpc.go
@@ -166,6 +166,8 @@ func testGetTransaction(t *testing.T, h *TestHandler) {
 			continue
 		}
 		got.Confirmations = 0
+		// CoinSpecificData are not specified in the fixtures
+		got.CoinSpecificData = nil
 
 		if !reflect.DeepEqual(got, want) {
 			t.Errorf("GetTransaction() got %+v, want %+v", got, want)
@@ -218,7 +220,7 @@ func testMempoolSync(t *testing.T, h *TestHandler) {
 				if err != nil {
 					t.Fatalf("address %q: %s", a, err)
 				}
-				if !containsString(got, txid) {
+				if !containsTx(got, txid) {
 					t.Errorf("ResyncMempool() - for address %s, transaction %s wasn't found in mempool", a, txid)
 					return
 				}
@@ -399,9 +401,9 @@ func intersect(a, b []string) []string {
 	return res
 }
 
-func containsString(slice []string, s string) bool {
-	for i := range slice {
-		if slice[i] == s {
+func containsTx(o []bchain.Outpoint, tx string) bool {
+	for i := range o {
+		if o[i].Txid == tx {
 			return true
 		}
 	}
diff --git a/tests/sync/connectblocks.go b/tests/sync/connectblocks.go
index 50d91ca0..36437192 100644
--- a/tests/sync/connectblocks.go
+++ b/tests/sync/connectblocks.go
@@ -107,7 +107,7 @@ func verifyBlockInfo(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range) {
 func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range) {
 	type txInfo struct {
 		txid     string
-		vout     uint32
+		vout     int32
 		isOutput bool
 	}
 	addr2txs := make(map[string][]txInfo)
@@ -122,13 +122,13 @@ func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range)
 		for _, tx := range block.TxDetails {
 			for _, vin := range tx.Vin {
 				for _, a := range vin.Addresses {
-					addr2txs[a] = append(addr2txs[a], txInfo{tx.Txid, vin.Vout, false})
+					addr2txs[a] = append(addr2txs[a], txInfo{tx.Txid, int32(vin.Vout), false})
 					checkMap[a] = append(checkMap[a], false)
 				}
 			}
 			for _, vout := range tx.Vout {
 				for _, a := range vout.ScriptPubKey.Addresses {
-					addr2txs[a] = append(addr2txs[a], txInfo{tx.Txid, vout.N, true})
+					addr2txs[a] = append(addr2txs[a], txInfo{tx.Txid, int32(vout.N), true})
 					checkMap[a] = append(checkMap[a], false)
 				}
 			}
@@ -136,7 +136,7 @@ func verifyTransactions(t *testing.T, d *db.RocksDB, h *TestHandler, rng Range)
 	}
 
 	for addr, txs := range addr2txs {
-		err := d.GetTransactions(addr, rng.Lower, rng.Upper, func(txid string, vout uint32, isOutput bool) error {
+		err := d.GetTransactions(addr, rng.Lower, rng.Upper, func(txid string, vout int32, isOutput bool) error {
 			for i, tx := range txs {
 				if txid == tx.txid && vout == tx.vout && isOutput == tx.isOutput {
 					checkMap[addr][i] = true
diff --git a/tests/sync/handlefork.go b/tests/sync/handlefork.go
index f51d03a2..2ed99cfa 100644
--- a/tests/sync/handlefork.go
+++ b/tests/sync/handlefork.go
@@ -129,7 +129,7 @@ func verifyTransactions2(t *testing.T, d *db.RocksDB, rng Range, addr2txs map[st
 			checkMap[txid] = false
 		}
 
-		err := d.GetTransactions(addr, rng.Lower, rng.Upper, func(txid string, vout uint32, isOutput bool) error {
+		err := d.GetTransactions(addr, rng.Lower, rng.Upper, func(txid string, vout int32, isOutput bool) error {
 			if isOutput {
 				checkMap[txid] = true
 			}