From c374ef86fd2801d341d556b9e200cf33b0395996 Mon Sep 17 00:00:00 2001 From: Martin Boehm Date: Fri, 10 Dec 2021 00:30:27 +0100 Subject: [PATCH] Update types in preparation for eth internal transactions, bump dbVersion to 6 --- api/types.go | 14 ++-- bchain/coins/eth/erc20.go | 4 +- bchain/coins/eth/erc20_test.go | 12 +-- bchain/coins/eth/ethparser.go | 62 ++++----------- bchain/coins/eth/ethparser_test.go | 36 ++++----- bchain/coins/eth/ethrpc.go | 18 ++--- bchain/types.go | 43 ----------- bchain/types_ethereumtype.go | 85 +++++++++++++++++++++ db/rocksdb.go | 6 +- db/rocksdb_ethereumtype.go | 8 +- db/rocksdb_ethereumtype_test.go | 24 +++--- docs/rocksdb.md | 4 +- tests/dbtestdata/dbtestdata_ethereumtype.go | 23 +++++- 13 files changed, 185 insertions(+), 154 deletions(-) create mode 100644 bchain/types_ethereumtype.go diff --git a/api/types.go b/api/types.go index eb0fad76..ce1ecd47 100644 --- a/api/types.go +++ b/api/types.go @@ -173,12 +173,14 @@ type TokenTransfer struct { // EthereumSpecific contains ethereum specific transaction data type EthereumSpecific struct { - Status eth.TxStatus `json:"status"` // 1 OK, 0 Fail, -1 pending - Nonce uint64 `json:"nonce"` - GasLimit *big.Int `json:"gasLimit"` - GasUsed *big.Int `json:"gasUsed"` - GasPrice *Amount `json:"gasPrice"` - Data string `json:"data,omitempty"` + TxType string `json:"txType,omitempty"` + Status eth.TxStatus `json:"status"` // 1 OK, 0 Fail, -1 pending + Nonce uint64 `json:"nonce"` + GasLimit *big.Int `json:"gasLimit"` + GasUsed *big.Int `json:"gasUsed"` + GasPrice *Amount `json:"gasPrice"` + Data string `json:"data,omitempty"` + InternalTransfers []bchain.EthereumInternalTransfer `json:"internalTransfers,omitempty"` } // Tx holds information about a transaction diff --git a/bchain/coins/eth/erc20.go b/bchain/coins/eth/erc20.go index d660511a..6971f707 100644 --- a/bchain/coins/eth/erc20.go +++ b/bchain/coins/eth/erc20.go @@ -56,7 +56,7 @@ func addressFromPaddedHex(s string) (string, error) { return a.String(), nil } -func erc20GetTransfersFromLog(logs []*rpcLog) ([]bchain.Erc20Transfer, error) { +func erc20GetTransfersFromLog(logs []*bchain.RpcLog) ([]bchain.Erc20Transfer, error) { var r []bchain.Erc20Transfer for _, l := range logs { if len(l.Topics) == 3 && l.Topics[0] == erc20TransferEventSignature { @@ -84,7 +84,7 @@ func erc20GetTransfersFromLog(logs []*rpcLog) ([]bchain.Erc20Transfer, error) { return r, nil } -func erc20GetTransfersFromTx(tx *rpcTransaction) ([]bchain.Erc20Transfer, error) { +func erc20GetTransfersFromTx(tx *bchain.RpcTransaction) ([]bchain.Erc20Transfer, error) { var r []bchain.Erc20Transfer if len(tx.Payload) == 128+len(erc20TransferMethodSignature) && strings.HasPrefix(tx.Payload, erc20TransferMethodSignature) { to, err := addressFromPaddedHex(tx.Payload[len(erc20TransferMethodSignature) : 64+len(erc20TransferMethodSignature)]) diff --git a/bchain/coins/eth/erc20_test.go b/bchain/coins/eth/erc20_test.go index f0a584f9..574144d4 100644 --- a/bchain/coins/eth/erc20_test.go +++ b/bchain/coins/eth/erc20_test.go @@ -15,13 +15,13 @@ import ( func TestErc20_erc20GetTransfersFromLog(t *testing.T) { tests := []struct { name string - args []*rpcLog + args []*bchain.RpcLog want []bchain.Erc20Transfer wantErr bool }{ { name: "1", - args: []*rpcLog{ + args: []*bchain.RpcLog{ { Address: "0x76a45e8976499ab9ae223cc584019341d5a84e96", Topics: []string{ @@ -43,7 +43,7 @@ func TestErc20_erc20GetTransfersFromLog(t *testing.T) { }, { name: "2", - args: []*rpcLog{ + args: []*bchain.RpcLog{ { // Transfer Address: "0x0d0f936ee4c93e25944694d6c121de94d9760f11", Topics: []string{ @@ -167,17 +167,17 @@ func TestErc20_erc20GetTransfersFromTx(t *testing.T) { bn, _ := new(big.Int).SetString("21e19e0c9bab2400000", 16) tests := []struct { name string - args *rpcTransaction + args *bchain.RpcTransaction want []bchain.Erc20Transfer }{ { name: "0", - args: (b.Txs[0].CoinSpecificData.(completeTransaction)).Tx, + args: (b.Txs[0].CoinSpecificData.(bchain.EthereumSpecificData)).Tx, want: []bchain.Erc20Transfer{}, }, { name: "1", - args: (b.Txs[1].CoinSpecificData.(completeTransaction)).Tx, + args: (b.Txs[1].CoinSpecificData.(bchain.EthereumSpecificData)).Tx, want: []bchain.Erc20Transfer{ { Contract: "0x4af4114f73d1c1c903ac9e0361b379d1291808a2", diff --git a/bchain/coins/eth/ethparser.go b/bchain/coins/eth/ethparser.go index 8526825d..8b8bdf1d 100644 --- a/bchain/coins/eth/ethparser.go +++ b/bchain/coins/eth/ethparser.go @@ -41,49 +41,13 @@ type rpcHeader struct { Nonce string `json:"nonce"` } -type rpcTransaction struct { - AccountNonce string `json:"nonce"` - GasPrice string `json:"gasPrice"` - GasLimit string `json:"gas"` - To string `json:"to"` // nil means contract creation - Value string `json:"value"` - Payload string `json:"input"` - Hash string `json:"hash"` - BlockNumber string `json:"blockNumber"` - BlockHash string `json:"blockHash,omitempty"` - From string `json:"from"` - TransactionIndex string `json:"transactionIndex"` - // Signature values - ignored - // V string `json:"v"` - // R string `json:"r"` - // S string `json:"s"` -} - -type rpcLog struct { - Address string `json:"address"` - Topics []string `json:"topics"` - Data string `json:"data"` -} - type rpcLogWithTxHash struct { - rpcLog + bchain.RpcLog Hash string `json:"transactionHash"` } -type rpcReceipt struct { - GasUsed string `json:"gasUsed"` - Status string `json:"status"` - Logs []*rpcLog `json:"logs"` -} - -type completeTransaction struct { - Tx *rpcTransaction `json:"tx"` - InternalData *bchain.EthereumInternalData `json:"internalData,omitempty"` - Receipt *rpcReceipt `json:"receipt,omitempty"` -} - type rpcBlockTransactions struct { - Transactions []rpcTransaction `json:"transactions"` + Transactions []bchain.RpcTransaction `json:"transactions"` } type rpcBlockTxids struct { @@ -97,7 +61,7 @@ func ethNumber(n string) (int64, error) { return 0, errors.Errorf("Not a number: '%v'", n) } -func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, internalData *bchain.EthereumInternalData, blocktime int64, confirmations uint32, fixEIP55 bool) (*bchain.Tx, error) { +func (p *EthereumParser) ethTxToTx(tx *bchain.RpcTransaction, receipt *bchain.RpcReceipt, internalData *bchain.EthereumInternalData, blocktime int64, confirmations uint32, fixEIP55 bool) (*bchain.Tx, error) { txid := tx.Hash var ( fa, ta []string @@ -136,7 +100,7 @@ func (p *EthereumParser) ethTxToTx(tx *rpcTransaction, receipt *rpcReceipt, inte } } } - ct := completeTransaction{ + ct := bchain.EthereumSpecificData{ Tx: tx, InternalData: internalData, Receipt: receipt, @@ -274,7 +238,7 @@ func hexEncodeBig(b []byte) string { func (p *EthereumParser) PackTx(tx *bchain.Tx, height uint32, blockTime int64) ([]byte, error) { var err error var n uint64 - r, ok := tx.CoinSpecificData.(completeTransaction) + r, ok := tx.CoinSpecificData.(bchain.EthereumSpecificData) if !ok { return nil, errors.New("Missing CoinSpecificData") } @@ -373,7 +337,7 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { if err != nil { return nil, 0, err } - rt := rpcTransaction{ + rt := bchain.RpcTransaction{ AccountNonce: hexutil.EncodeUint64(pt.Tx.AccountNonce), BlockNumber: hexutil.EncodeUint64(uint64(pt.BlockNumber)), From: EIP55Address(pt.Tx.From), @@ -388,15 +352,15 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { TransactionIndex: hexutil.EncodeUint64(uint64(pt.Tx.TransactionIndex)), Value: hexEncodeBig(pt.Tx.Value), } - var rr *rpcReceipt + var rr *bchain.RpcReceipt if pt.Receipt != nil { - logs := make([]*rpcLog, len(pt.Receipt.Log)) + logs := make([]*bchain.RpcLog, len(pt.Receipt.Log)) for i, l := range pt.Receipt.Log { topics := make([]string, len(l.Topics)) for j, t := range l.Topics { topics[j] = hexutil.Encode(t) } - logs[i] = &rpcLog{ + logs[i] = &bchain.RpcLog{ Address: EIP55Address(l.Address), Data: hexutil.Encode(l.Data), Topics: topics, @@ -407,7 +371,7 @@ func (p *EthereumParser) UnpackTx(buf []byte) (*bchain.Tx, uint32, error) { if len(pt.Receipt.Status) != 1 || pt.Receipt.Status[0] != 'U' { status = hexEncodeBig(pt.Receipt.Status) } - rr = &rpcReceipt{ + rr = &bchain.RpcReceipt{ GasUsed: hexEncodeBig(pt.Receipt.GasUsed), Status: status, Logs: logs, @@ -460,7 +424,7 @@ func (p *EthereumParser) GetChainType() bchain.ChainType { // GetHeightFromTx returns ethereum specific data from bchain.Tx func GetHeightFromTx(tx *bchain.Tx) (uint32, error) { var bn string - csd, ok := tx.CoinSpecificData.(completeTransaction) + csd, ok := tx.CoinSpecificData.(bchain.EthereumSpecificData) if !ok { return 0, errors.New("Missing CoinSpecificData") } @@ -476,7 +440,7 @@ func GetHeightFromTx(tx *bchain.Tx) (uint32, error) { func (p *EthereumParser) EthereumTypeGetErc20FromTx(tx *bchain.Tx) ([]bchain.Erc20Transfer, error) { var r []bchain.Erc20Transfer var err error - csd, ok := tx.CoinSpecificData.(completeTransaction) + csd, ok := tx.CoinSpecificData.(bchain.EthereumSpecificData) if ok { if csd.Receipt != nil { r, err = erc20GetTransfersFromLog(csd.Receipt.Logs) @@ -519,7 +483,7 @@ func GetEthereumTxData(tx *bchain.Tx) *EthereumTxData { // GetEthereumTxDataFromSpecificData returns EthereumTxData from coinSpecificData func GetEthereumTxDataFromSpecificData(coinSpecificData interface{}) *EthereumTxData { etd := EthereumTxData{Status: TxStatusPending} - csd, ok := coinSpecificData.(completeTransaction) + csd, ok := coinSpecificData.(bchain.EthereumSpecificData) if ok { if csd.Tx != nil { etd.Nonce, _ = hexutil.DecodeUint64(csd.Tx.AccountNonce) diff --git a/bchain/coins/eth/ethparser_test.go b/bchain/coins/eth/ethparser_test.go index a9e29703..c990343c 100644 --- a/bchain/coins/eth/ethparser_test.go +++ b/bchain/coins/eth/ethparser_test.go @@ -89,8 +89,8 @@ func init() { }, }, }, - CoinSpecificData: completeTransaction{ - Tx: &rpcTransaction{ + CoinSpecificData: bchain.EthereumSpecificData{ + Tx: &bchain.RpcTransaction{ AccountNonce: "0xb26c", GasPrice: "0x430e23400", GasLimit: "0x5208", @@ -102,10 +102,10 @@ func init() { From: "0x3E3a3D69dc66bA10737F531ed088954a9EC89d97", TransactionIndex: "0xa", }, - Receipt: &rpcReceipt{ + Receipt: &bchain.RpcReceipt{ GasUsed: "0x5208", Status: "0x1", - Logs: []*rpcLog{}, + Logs: []*bchain.RpcLog{}, }, }, } @@ -127,8 +127,8 @@ func init() { }, }, }, - CoinSpecificData: completeTransaction{ - Tx: &rpcTransaction{ + CoinSpecificData: bchain.EthereumSpecificData{ + Tx: &bchain.RpcTransaction{ AccountNonce: "0xd0", GasPrice: "0x9502f9000", GasLimit: "0x130d5", @@ -139,10 +139,10 @@ func init() { BlockNumber: "0x41eee8", From: "0x20cD153de35D469BA46127A0C8F18626b59a256A", TransactionIndex: "0x0"}, - Receipt: &rpcReceipt{ + Receipt: &bchain.RpcReceipt{ GasUsed: "0xcb39", Status: "0x1", - Logs: []*rpcLog{ + Logs: []*bchain.RpcLog{ { Address: "0x4af4114F73d1c1C903aC9E0361b379D1291808A2", Data: "0x00000000000000000000000000000000000000000000021e19e0c9bab2400000", @@ -174,8 +174,8 @@ func init() { }, }, }, - CoinSpecificData: completeTransaction{ - Tx: &rpcTransaction{ + CoinSpecificData: bchain.EthereumSpecificData{ + Tx: &bchain.RpcTransaction{ AccountNonce: "0xb26c", GasPrice: "0x430e23400", GasLimit: "0x5208", @@ -187,10 +187,10 @@ func init() { From: "0x3E3a3D69dc66bA10737F531ed088954a9EC89d97", TransactionIndex: "0xa", }, - Receipt: &rpcReceipt{ + Receipt: &bchain.RpcReceipt{ GasUsed: "0x5208", Status: "0x0", - Logs: []*rpcLog{}, + Logs: []*bchain.RpcLog{}, }, }, } @@ -212,8 +212,8 @@ func init() { }, }, }, - CoinSpecificData: completeTransaction{ - Tx: &rpcTransaction{ + CoinSpecificData: bchain.EthereumSpecificData{ + Tx: &bchain.RpcTransaction{ AccountNonce: "0xb26c", GasPrice: "0x430e23400", GasLimit: "0x5208", @@ -225,10 +225,10 @@ func init() { From: "0x3E3a3D69dc66bA10737F531ed088954a9EC89d97", TransactionIndex: "0xa", }, - Receipt: &rpcReceipt{ + Receipt: &bchain.RpcReceipt{ GasUsed: "0x5208", Status: "", - Logs: []*rpcLog{}, + Logs: []*bchain.RpcLog{}, }, }, } @@ -351,8 +351,8 @@ func TestEthereumParser_UnpackTx(t *testing.T) { return } // DeepEqual has problems with pointers in completeTransaction - gs := got.CoinSpecificData.(completeTransaction) - ws := tt.want.CoinSpecificData.(completeTransaction) + gs := got.CoinSpecificData.(bchain.EthereumSpecificData) + ws := tt.want.CoinSpecificData.(bchain.EthereumSpecificData) gc := *got wc := *tt.want gc.CoinSpecificData = nil diff --git a/bchain/coins/eth/ethrpc.go b/bchain/coins/eth/ethrpc.go index 8bb585aa..da3e02d8 100644 --- a/bchain/coins/eth/ethrpc.go +++ b/bchain/coins/eth/ethrpc.go @@ -490,7 +490,7 @@ func (b *EthereumRPC) getBlockRaw(hash string, height uint32, fullTxs bool) (jso return raw, nil } -func (b *EthereumRPC) getERC20EventsForBlock(blockNumber string) (map[string][]*rpcLog, error) { +func (b *EthereumRPC) getERC20EventsForBlock(blockNumber string) (map[string][]*bchain.RpcLog, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() var logs []rpcLogWithTxHash @@ -502,10 +502,10 @@ func (b *EthereumRPC) getERC20EventsForBlock(blockNumber string) (map[string][]* if err != nil { return nil, errors.Annotatef(err, "blockNumber %v", blockNumber) } - r := make(map[string][]*rpcLog) + r := make(map[string][]*bchain.RpcLog) for i := range logs { l := &logs[i] - r[l.Hash] = append(r[l.Hash], &l.rpcLog) + r[l.Hash] = append(r[l.Hash], &l.RpcLog) } return r, nil } @@ -555,7 +555,7 @@ func (b *EthereumRPC) processCallTrace(call rpcCallTrace, d *bchain.EthereumInte // getInternalDataForBlock fetches debug trace using callTracer, extracts internal transfers and creations and destructions of contracts // by design, it never returns error so that missing internal transactions do not stop the rest of the blockchain import -func (b *EthereumRPC) getInternalDataForBlock(blockHash string, transactions []rpcTransaction) ([]bchain.EthereumInternalData, error) { +func (b *EthereumRPC) getInternalDataForBlock(blockHash string, transactions []bchain.RpcTransaction) ([]bchain.EthereumInternalData, error) { data := make([]bchain.EthereumInternalData, len(transactions)) if b.ChainConfig.ProcessInternalTransactions { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) @@ -619,7 +619,7 @@ func (b *EthereumRPC) GetBlock(hash string, height uint32) (*bchain.Block, error btxs := make([]bchain.Tx, len(body.Transactions)) for i := range body.Transactions { tx := &body.Transactions[i] - btx, err := b.Parser.ethTxToTx(tx, &rpcReceipt{Logs: logs[tx.Hash]}, &internalData[i], bbh.Time, uint32(bbh.Confirmations), true) + btx, err := b.Parser.ethTxToTx(tx, &bchain.RpcReceipt{Logs: logs[tx.Hash]}, &internalData[i], bbh.Time, uint32(bbh.Confirmations), true) if err != nil { return nil, errors.Annotatef(err, "hash %v, height %v, txid %v", hash, height, tx.Hash) } @@ -671,7 +671,7 @@ func (b *EthereumRPC) GetTransactionForMempool(txid string) (*bchain.Tx, error) func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() - var tx *rpcTransaction + var tx *bchain.RpcTransaction hash := ethcommon.HexToHash(txid) err := b.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash) if err != nil { @@ -705,7 +705,7 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { if time, err = ethNumber(ht.Time); err != nil { return nil, errors.Annotatef(err, "txid %v", txid) } - var receipt rpcReceipt + var receipt bchain.RpcReceipt err = b.rpc.CallContext(ctx, &receipt, "eth_getTransactionReceipt", hash) if err != nil { return nil, errors.Annotatef(err, "txid %v", txid) @@ -733,13 +733,13 @@ func (b *EthereumRPC) GetTransaction(txid string) (*bchain.Tx, error) { // GetTransactionSpecific returns json as returned by backend, with all coin specific data func (b *EthereumRPC) GetTransactionSpecific(tx *bchain.Tx) (json.RawMessage, error) { - csd, ok := tx.CoinSpecificData.(completeTransaction) + csd, ok := tx.CoinSpecificData.(bchain.EthereumSpecificData) if !ok { ntx, err := b.GetTransaction(tx.Txid) if err != nil { return nil, err } - csd, ok = ntx.CoinSpecificData.(completeTransaction) + csd, ok = ntx.CoinSpecificData.(bchain.EthereumSpecificData) if !ok { return nil, errors.New("Cannot get CoinSpecificData") } diff --git a/bchain/types.go b/bchain/types.go index 7c96f773..555db84a 100644 --- a/bchain/types.go +++ b/bchain/types.go @@ -200,49 +200,6 @@ func AddressDescriptorFromString(s string) (AddressDescriptor, error) { return nil, errors.New("invalid address descriptor") } -// EthereumType specific - -// EthereumInternalTransfer contains data about internal transfer -type EthereumInternalTransfer struct { - Type EthereumInternalTransactionType `json:"type"` - From string `json:"from"` - To string `json:"to"` - Value big.Int `json:"value"` -} - -// EthereumInternalTransactionType - type of ethereum transaction from internal data -type EthereumInternalTransactionType int - -// EthereumInternalTransactionType enumeration -const ( - CALL = EthereumInternalTransactionType(iota) - CREATE - SELFDESTRUCT -) - -// EthereumInternalTransaction contains internal transfers -type EthereumInternalData struct { - Type EthereumInternalTransactionType `json:"type"` - Contract string `json:"contract,omitempty"` - Transfers []EthereumInternalTransfer `json:"transfers,omitempty"` -} - -// Erc20Contract contains info about ERC20 contract -type Erc20Contract struct { - Contract string `json:"contract"` - Name string `json:"name"` - Symbol string `json:"symbol"` - Decimals int `json:"decimals"` -} - -// Erc20Transfer contains a single ERC20 token transfer -type Erc20Transfer struct { - Contract string - From string - To string - Tokens big.Int -} - // MempoolTxidEntry contains mempool txid with first seen time type MempoolTxidEntry struct { Txid string diff --git a/bchain/types_ethereumtype.go b/bchain/types_ethereumtype.go new file mode 100644 index 00000000..94e4ecc2 --- /dev/null +++ b/bchain/types_ethereumtype.go @@ -0,0 +1,85 @@ +package bchain + +import "math/big" + +// EthereumType specific + +// EthereumInternalTransfer contains data about internal transfer +type EthereumInternalTransfer struct { + Type EthereumInternalTransactionType `json:"type"` + From string `json:"from"` + To string `json:"to"` + Value big.Int `json:"value"` +} + +// EthereumInternalTransactionType - type of ethereum transaction from internal data +type EthereumInternalTransactionType int + +// EthereumInternalTransactionType enumeration +const ( + CALL = EthereumInternalTransactionType(iota) + CREATE + SELFDESTRUCT +) + +// EthereumInternalTransaction contains internal transfers +type EthereumInternalData struct { + Type EthereumInternalTransactionType `json:"type"` + Contract string `json:"contract,omitempty"` + Transfers []EthereumInternalTransfer `json:"transfers,omitempty"` +} + +// Erc20Contract contains info about ERC20 contract +type Erc20Contract struct { + Contract string `json:"contract"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals int `json:"decimals"` +} + +// Erc20Transfer contains a single ERC20 token transfer +type Erc20Transfer struct { + Contract string + From string + To string + Tokens big.Int +} + +// RpcTransaction is returned by eth_getTransactionByHash +type RpcTransaction struct { + AccountNonce string `json:"nonce"` + GasPrice string `json:"gasPrice"` + GasLimit string `json:"gas"` + To string `json:"to"` // nil means contract creation + Value string `json:"value"` + Payload string `json:"input"` + Hash string `json:"hash"` + BlockNumber string `json:"blockNumber"` + BlockHash string `json:"blockHash,omitempty"` + From string `json:"from"` + TransactionIndex string `json:"transactionIndex"` + // Signature values - ignored + // V string `json:"v"` + // R string `json:"r"` + // S string `json:"s"` +} + +// RpcLog is returned by eth_getLogs +type RpcLog struct { + Address string `json:"address"` + Topics []string `json:"topics"` + Data string `json:"data"` +} + +// RpcLog is returned by eth_getTransactionReceipt +type RpcReceipt struct { + GasUsed string `json:"gasUsed"` + Status string `json:"status"` + Logs []*RpcLog `json:"logs"` +} + +type EthereumSpecificData struct { + Tx *RpcTransaction `json:"tx"` + InternalData *EthereumInternalData `json:"internalData,omitempty"` + Receipt *RpcReceipt `json:"receipt,omitempty"` +} diff --git a/db/rocksdb.go b/db/rocksdb.go index a03a2a1e..59b90c9a 100644 --- a/db/rocksdb.go +++ b/db/rocksdb.go @@ -22,7 +22,7 @@ import ( "github.com/trezor/blockbook/common" ) -const dbVersion = 5 +const dbVersion = 6 const packedHeightBytes = 4 const maxAddrDescLen = 1024 @@ -112,6 +112,8 @@ const ( cfTxAddresses // EthereumType cfAddressContracts = cfAddressBalance + cfInternalData + cfContracts ) // common columns @@ -120,7 +122,7 @@ var cfBaseNames = []string{"default", "height", "addresses", "blockTxs", "transa // type specific columns var cfNamesBitcoinType = []string{"addressBalance", "txAddresses"} -var cfNamesEthereumType = []string{"addressContracts"} +var cfNamesEthereumType = []string{"addressContracts", "internalData", "contracts"} func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { // opts with bloom filter diff --git a/db/rocksdb_ethereumtype.go b/db/rocksdb_ethereumtype.go index f1df45ed..e63b144c 100644 --- a/db/rocksdb_ethereumtype.go +++ b/db/rocksdb_ethereumtype.go @@ -22,6 +22,7 @@ type AddrContract struct { type AddrContracts struct { TotalTxs uint NonContractTxs uint + InternalTxs uint Contracts []AddrContract } @@ -30,7 +31,7 @@ func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string varBuf := make([]byte, vlq.MaxLen64) for addrDesc, acs := range acm { // address with 0 contracts is removed from db - happens on disconnect - if acs == nil || (acs.NonContractTxs == 0 && len(acs.Contracts) == 0) { + if acs == nil || (acs.NonContractTxs == 0 && acs.InternalTxs == 0 && len(acs.Contracts) == 0) { wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc)) } else { buf = buf[:0] @@ -38,6 +39,8 @@ func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string buf = append(buf, varBuf[:l]...) l = packVaruint(acs.NonContractTxs, varBuf) buf = append(buf, varBuf[:l]...) + l = packVaruint(acs.InternalTxs, varBuf) + buf = append(buf, varBuf[:l]...) for _, ac := range acs.Contracts { buf = append(buf, ac.Contract...) l = packVaruint(ac.Txs, varBuf) @@ -64,6 +67,8 @@ func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*Addr buf = buf[l:] nct, l := unpackVaruint(buf) buf = buf[l:] + ict, l := unpackVaruint(buf) + buf = buf[l:] c := make([]AddrContract, 0, 4) for len(buf) > 0 { if len(buf) < eth.EthereumTypeAddressDescriptorLen { @@ -80,6 +85,7 @@ func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*Addr return &AddrContracts{ TotalTxs: tt, NonContractTxs: nct, + InternalTxs: ict, Contracts: c, }, nil } diff --git a/db/rocksdb_ethereumtype_test.go b/db/rocksdb_ethereumtype_test.go index 9819ec5d..55af30ff 100644 --- a/db/rocksdb_ethereumtype_test.go +++ b/db/rocksdb_ethereumtype_test.go @@ -44,10 +44,10 @@ func verifyAfterEthereumTypeBlock1(t *testing.T, d *RocksDB, afterDisconnect boo } if err := checkColumn(d, cfAddressContracts, []keyPair{ - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "0101", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "0201" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "0101" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "0101", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "010100", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "020100" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "010100" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "010100", nil}, }); err != nil { { t.Fatal(err) @@ -113,14 +113,14 @@ func verifyAfterEthereumTypeBlock2(t *testing.T, d *RocksDB) { } if err := checkColumn(d, cfAddressContracts, []keyPair{ - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "0101", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "0402" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "0101" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "0101", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "0101", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "0101" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "0100" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, - {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "0101", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr3e, d.chainParser), "010100", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr55, d.chainParser), "040200" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr20, d.chainParser), "010100" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser), "010100", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr9f, d.chainParser), "010100", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr4b, d.chainParser), "010100" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "02" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "02", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddr7b, d.chainParser), "010000" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract4a, d.chainParser) + "01" + dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract0d, d.chainParser) + "01", nil}, + {dbtestdata.AddressToPubKeyHex(dbtestdata.EthAddrContract47, d.chainParser), "010100", nil}, }); err != nil { { t.Fatal(err) diff --git a/docs/rocksdb.md b/docs/rocksdb.md index 4301b0f7..3860a70b 100644 --- a/docs/rocksdb.md +++ b/docs/rocksdb.md @@ -84,9 +84,9 @@ Column families used only by **Ethereum type** coins: - **addressContracts** (used only by Ethereum type coins) - Maps *addrDesc* to *total number of transactions*, *number of non contract transactions* and array of *contracts* with *number of transfers* of given address. + Maps *addrDesc* to *total number of transactions*, *number of non contract transactions*, *number of internal transactions* and array of *contracts* with *number of transfers* of given address. ``` - (addrDesc []byte) -> (total_txs vuint)+(non-contract_txs vuint)+[]((contractAddrDesc []byte)+(nr_transfers vuint)) + (addrDesc []byte) -> (total_txs vuint)+(non-contract_txs vuint)+(internal_txs vuint)+[]((contractAddrDesc []byte)+(nr_transfers vuint)) ``` - **blockTxs** diff --git a/tests/dbtestdata/dbtestdata_ethereumtype.go b/tests/dbtestdata/dbtestdata_ethereumtype.go index 085eb764..cebc9cb2 100644 --- a/tests/dbtestdata/dbtestdata_ethereumtype.go +++ b/tests/dbtestdata/dbtestdata_ethereumtype.go @@ -30,10 +30,15 @@ const ( EthTx4Packed = "08e9dd870210d4b5f0db051aa50b08f6be0712043b9aca001890a10f2ac40a4f15078700000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000000000000000000000000000000000000000022000000000000000000000000000000000000000000000000000000000000003c00000000000000000000000000000000000000000000000000000000000000420000000000000000000000000000000000000000000000000000000000000048000000000000000000000000000000000000000000000000000000000000004e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a200000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b0000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000a5ef5a7656bfb0000000000000000000000000000000000000000000000000000004ba78398d5c5000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfe0b9579b4ecf7a2801880f644009a324671a79754ea57c3a103c6e70d3dbef6ba69a08000000000000000000000000000000000000000000000000004f937d86afb90000000000000000000000000000000000000000000000000ab280fd8037d500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000166cfb784b7c1f3fbe8b75484603ab8adc58aaee3a46245a6579fac7077b5570018b4e0d4eb0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000308fd0e798ac00000000000000000000000000000000000000000000000006a8313d60b1f80000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000001b000000000000000000000000000000000000000000000000000000000000001b00000000000000000000000000000000000000000000000000000000000000029de0ccec59e8948e3d905b40e5542335ebc1eb4674db517d2f6392ec7fdeb3d45f3449d313ee2589819c6c79eb1c1b047adae68565c1608e3a1d1d70823febb0000000000000000000000000000000000000000000000000000000000000000234d06fe17f1202e8b07177a30eb64d14adc08cdb3fa1b3e3e0bea0f9672c02175b77c01c51d3c7e460723b27ecbc7801fd6482559a8c9999593f9a4d149c73843220c92919ad24ffd58f760b18df7949f06e1190cf54a50a0e3745a385608ed3cbf23a14479cc461fecd078f766ecc58533d6f69580cf3ac42144bda106325c335df99eab7fe363cac8a0ba2a24d482422d40b0a03034d301201011a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f606b1a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a21220000000000000000000000000000000000000000000000000000308fd0e798ac01a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f110000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000000000000000000000000006a8313d60b1f606b000000000000000000000000000000000000000000000000000308fd0e798ac0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005e083a16f4b092c5729a49f9c3ed3cc171bb3d3d0c22e20b1de6063c32f399ac1a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a20000000000000000000000000555ee11fbddc0e49a9bab358a8941ad95ffdb48f1a2000000000000000000000000000000000000000000000000000000000000000001a205af266c0a89a07c1917deaa024414577e6c3c31c8907d079e13eb448c082594f1a9e010a144af4114f73d1c1c903ac9e0361b379d1291808a2122000000000000000000000000000000000000000000000000000031855667df7a81a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a9e010a140d0f936ee4c93e25944694d6c121de94d9760f1112200000000000000000000000000000000000000000000000006a8313d60b1f80001a20ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef1a200000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d1a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1aa1030a14479cc461fecd078f766ecc58533d6f69580cf3ac1280020000000000000000000000004bda106325c335df99eab7fe363cac8a0ba2a24d0000000000000000000000004af4114f73d1c1c903ac9e0361b379d1291808a20000000000000000000000000d0f936ee4c93e25944694d6c121de94d9760f1100000000000000000000000000000000000000000000000000031855667df7a80000000000000000000000000000000000000000000000006a8313d60b1f800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f2b0d62c44ed08f2a5adef40c875d20310a42a9d4f488bd26323256fe01c7f481a200d0b9391970d9a25552f37d436d2aae2925e2bfe1b2a923754bada030c498cb31a200000000000000000000000007b62eb7fe80350dc7ec945c0b73242cb9877fb1b1a2000000000000000000000000000000000000000000000000000000000000000001a20b0b69dad58df6032c3b266e19b1045b19c87acd2c06fb0c598090f44b8e263aa" ) -func unpackTxs(packed []string, parser bchain.BlockChainParser) []bchain.Tx { +type packedAndInternal struct { + packed string + internal *bchain.EthereumInternalData +} + +func unpackTxs(packed []packedAndInternal, parser bchain.BlockChainParser) []bchain.Tx { r := make([]bchain.Tx, len(packed)) for i, p := range packed { - b, err := hex.DecodeString(p) + b, err := hex.DecodeString(p.packed) if err != nil { panic(err) } @@ -41,6 +46,8 @@ func unpackTxs(packed []string, parser bchain.BlockChainParser) []bchain.Tx { if err != nil { panic(err) } + c, _ := tx.CoinSpecificData.(bchain.EthereumSpecificData) + c.InternalData = p.internal r[i] = *tx } return r @@ -56,7 +63,11 @@ func GetTestEthereumTypeBlock1(parser bchain.BlockChainParser) *bchain.Block { Time: 1534858022, Confirmations: 2, }, - Txs: unpackTxs([]string{EthTx1Packed, EthTx2Packed}, parser), + Txs: unpackTxs([]packedAndInternal{{ + packed: EthTx1Packed, + }, { + packed: EthTx2Packed, + }}, parser), } } @@ -70,6 +81,10 @@ func GetTestEthereumTypeBlock2(parser bchain.BlockChainParser) *bchain.Block { Time: 1534859988, Confirmations: 1, }, - Txs: unpackTxs([]string{EthTx3Packed, EthTx4Packed}, parser), + Txs: unpackTxs([]packedAndInternal{{ + packed: EthTx3Packed, + }, { + packed: EthTx4Packed, + }}, parser), } }