RanchiMall/blockbook
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Download ETH 4byte signatures

Browse Source
This commit is contained in:
Martin Boehm 2022-02-20 17:59:06 +01:00 committed by Martin
parent d5e871818a
commit f57bd2e6c3
12 changed files with 466 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bchain/coins/eth/dataparser.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/hex"
"math/big"
"unicode"
"unicode/utf8"
)
@ -56,3 +57,16 @@ func parseSimpleStringProperty(data string) string {
}
return ""
}
func Decamel(s string) string {
var b bytes.Buffer
splittable := false
for _, v := range s {
if splittable && unicode.IsUpper(v) {
b.WriteByte(' ')
}
b.WriteRune(v)
splittable = unicode.IsLower(v) || unicode.IsNumber(v)
}
return b.String()
}

30
blockbook.go
View File

@ -24,6 +24,7 @@ import (
"github.com/trezor/blockbook/common"
"github.com/trezor/blockbook/db"
"github.com/trezor/blockbook/fiat"
"github.com/trezor/blockbook/fourbyte"
"github.com/trezor/blockbook/server"
)
@ -343,7 +344,7 @@ func mainWithExitCode() int {
if internalServer != nil || publicServer != nil || chain != nil {
// start fiat rates downloader only if not shutting down immediately
initFiatRatesDownloader(index, *configFile)
initDownloaders(index, chain, *configFile)
waitForSignalAndShutdown(internalServer, publicServer, chain, 10*time.Second)
}
@ -667,7 +668,7 @@ func computeFeeStats(stopCompute chan os.Signal, blockFrom, blockTo int, db *db.
return err
}
func initFiatRatesDownloader(db *db.RocksDB, configfile string) {
func initDownloaders(db *db.RocksDB, chain bchain.BlockChain, configfile string) {
data, err := ioutil.ReadFile(configfile)
if err != nil {
glog.Errorf("Error reading file %v, %v", configfile, err)
@ -675,8 +676,9 @@ func initFiatRatesDownloader(db *db.RocksDB, configfile string) {
}
var config struct {
FiatRates string `json:"fiat_rates"`
FiatRatesParams string `json:"fiat_rates_params"`
FiatRates string `json:"fiat_rates"`
FiatRatesParams string `json:"fiat_rates_params"`
FourByteSignatures string `json:"fourByteSignatures"`
}
err = json.Unmarshal(data, &config)
@ -686,14 +688,26 @@ func initFiatRatesDownloader(db *db.RocksDB, configfile string) {
}
if config.FiatRates == "" || config.FiatRatesParams == "" {
glog.Infof("FiatRates config (%v) is empty, so the functionality is disabled.", configfile)
glog.Infof("FiatRates config (%v) is empty, not downloading fiat rates.", configfile)
} else {
fiatRates, err := fiat.NewFiatRatesDownloader(db, config.FiatRates, config.FiatRatesParams, nil, onNewFiatRatesTicker)
if err != nil {
glog.Errorf("NewFiatRatesDownloader Init error: %v", err)
return
} else {
glog.Infof("Starting %v FiatRates downloader...", config.FiatRates)
go fiatRates.Run()
}
glog.Infof("Starting %v FiatRates downloader...", config.FiatRates)
go fiatRates.Run()
}
if config.FourByteSignatures != "" && chain.GetChainParser().GetChainType() == bchain.ChainEthereumType {
fbsd, err := fourbyte.NewFourByteSignaturesDownloader(db, config.FourByteSignatures)
if err != nil {
glog.Errorf("NewFourByteSignaturesDownloader Init error: %v", err)
} else {
glog.Infof("Starting FourByteSignatures downloader...")
go fbsd.Run()
}
}
}

3
configs/coins/ethereum_archive.json
View File

@ -54,7 +54,8 @@
"processInternalTransactions": true,
"queryBackendOnMempoolResync": false,
"fiat_rates": "coingecko",
"fiat_rates_params": "{\"url\": \"https://api.coingecko.com/api/v3\", \"coin\": \"ethereum\", \"periodSeconds\": 60}"
"fiat_rates_params": "{\"url\": \"https://api.coingecko.com/api/v3\", \"coin\": \"ethereum\", \"periodSeconds\": 60}",
"4byteSignatures": "https://www.4byte.directory/api/v1/signatures/"
}
}
},

3
configs/coins/ethereum_testnet_ropsten_archive.json
View File

@ -51,7 +51,8 @@
"additional_params": {
"mempoolTxTimeoutHours": 12,
"processInternalTransactions": true,
"queryBackendOnMempoolResync": false
"queryBackendOnMempoolResync": false,
"fourByteSignatures": "https://www.4byte.directory/api/v1/signatures/"
}
}
},

15
db/bulkconnect.go
View File

@ -108,7 +108,7 @@ func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) {
c <- err
return
}
if err := b.d.db.Write(b.d.wo, wb); err != nil {
if err := b.d.WriteBatch(wb); err != nil {
c <- err
return
}
@ -148,7 +148,7 @@ func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) {
c <- err
return
}
if err := b.d.db.Write(b.d.wo, wb); err != nil {
if err := b.d.WriteBatch(wb); err != nil {
c <- err
return
}
@ -215,7 +215,7 @@ func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs
return err
}
}
if err := b.d.db.Write(b.d.wo, wb); err != nil {
if err := b.d.WriteBatch(wb); err != nil {
return err
}
if bac > b.bulkAddressesCount {
@ -267,7 +267,7 @@ func (b *BulkConnect) parallelStoreAddressContracts(c chan error, all bool) {
c <- err
return
}
if err := b.d.db.Write(b.d.wo, wb); err != nil {
if err := b.d.WriteBatch(wb); err != nil {
c <- err
return
}
@ -323,7 +323,7 @@ func (b *BulkConnect) connectBlockEthereumType(block *bchain.Block, storeBlockTx
return err
}
}
if err = b.d.db.Write(b.d.wo, wb); err != nil {
if err = b.d.WriteBatch(wb); err != nil {
return err
}
if bac > b.bulkAddressesCount {
@ -338,6 +338,9 @@ func (b *BulkConnect) connectBlockEthereumType(block *bchain.Block, storeBlockTx
if err = b.d.storeBlockSpecificDataEthereumType(wb, block); err != nil {
return err
}
if err := b.d.WriteBatch(wb); err != nil {
return err
}
}
}
if storeAddrContracts != nil {
@ -381,7 +384,7 @@ func (b *BulkConnect) Close() error {
if err := b.storeBulkAddresses(wb); err != nil {
return err
}
if err := b.d.db.Write(b.d.wo, wb); err != nil {
if err := b.d.WriteBatch(wb); err != nil {
return err
}
glog.Info("rocksdb: height ", b.height, ", stored ", bac, " addresses, done in ", time.Since(start))

31
db/rocksdb.go
View File

@ -196,6 +196,10 @@ func atoUint64(s string) uint64 {
return uint64(i)
}
func (d *RocksDB) WriteBatch(wb *gorocksdb.WriteBatch) error {
return d.db.Write(d.wo, wb)
}
// GetMemoryStats returns memory usage statistics as reported by RocksDB
func (d *RocksDB) GetMemoryStats() string {
var total, indexAndFilter, memtable uint64
@ -369,7 +373,7 @@ func (d *RocksDB) ConnectBlock(block *bchain.Block) error {
if err := d.storeAddresses(wb, block.Height, addresses); err != nil {
return err
}
if err := d.db.Write(d.wo, wb); err != nil {
if err := d.WriteBatch(wb); err != nil {
return err
}
d.is.AppendBlockTime(uint32(block.Time))
@ -1418,7 +1422,7 @@ func (d *RocksDB) disconnectBlock(height uint32, blockTxs []blockTxs) error {
wb.DeleteCF(d.cfh[cfTransactions], b)
wb.DeleteCF(d.cfh[cfTxAddresses], b)
}
return d.db.Write(d.wo, wb)
return d.WriteBatch(wb)
}
// DisconnectBlockRangeBitcoinType removes all data belonging to blocks in range lower-higher
@ -1535,7 +1539,7 @@ func (d *RocksDB) DeleteTx(txid string) error {
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
d.internalDeleteTx(wb, key)
return d.db.Write(d.wo, wb)
return d.WriteBatch(wb)
}
// internalDeleteTx checks if tx is cached and updates internal state accordingly
@ -1832,7 +1836,7 @@ func (d *RocksDB) fixUtxo(addrDesc bchain.AddressDescriptor, ba *AddrBalance) (b
wb := gorocksdb.NewWriteBatch()
err = d.storeBalances(wb, map[string]*AddrBalance{string(addrDesc): ba})
if err == nil {
err = d.db.Write(d.wo, wb)
err = d.WriteBatch(wb)
}
wb.Destroy()
if err != nil {
@ -1845,7 +1849,7 @@ func (d *RocksDB) fixUtxo(addrDesc bchain.AddressDescriptor, ba *AddrBalance) (b
wb := gorocksdb.NewWriteBatch()
err := d.storeBalances(wb, map[string]*AddrBalance{string(addrDesc): ba})
if err == nil {
err = d.db.Write(d.wo, wb)
err = d.WriteBatch(wb)
}
wb.Destroy()
if err != nil {
@ -1977,6 +1981,23 @@ func unpackVaruint(buf []byte) (uint, int) {
return uint(i), ofs
}
func packString(s string) []byte {
varBuf := make([]byte, vlq.MaxLen64)
l := len(s)
i := packVaruint(uint(l), varBuf)
buf := make([]byte, 0, i+l)
buf = append(buf, varBuf[:i]...)
buf = append(buf, s...)
return buf
}
func unpackString(buf []byte) (string, int) {
sl, l := unpackVaruint(buf)
so := l + int(sl)
s := string(buf[l:so])
return s, so
}
const (
// number of bits in a big.Word
wordBits = 32 << (uint64(^big.Word(0)) >> 63)

54
db/rocksdb_ethereumtype.go
View File

@ -578,6 +578,58 @@ func (d *RocksDB) unpackEthInternalData(buf []byte) (*bchain.EthereumInternalDat
return &id, nil
}
type FourByteSignature struct {
Name string
Parameters []string
}
func packFourByteKey(fourBytes uint32, id uint32) []byte {
key := make([]byte, 0, 8)
key = append(key, packUint(fourBytes)...)
key = append(key, packUint(id)...)
return key
}
func packFourByteSignature(signature *FourByteSignature) []byte {
buf := packString(signature.Name)
for i := range signature.Parameters {
buf = append(buf, packString(signature.Parameters[i])...)
}
return buf
}
func unpackFourByteSignature(buf []byte) (*FourByteSignature, error) {
var signature FourByteSignature
var l int
signature.Name, l = unpackString(buf)
for l < len(buf) {
s, ll := unpackString(buf[l:])
signature.Parameters = append(signature.Parameters, s)
l += ll
}
return &signature, nil
}
func (d *RocksDB) GetFourByteSignature(fourBytes uint32, id uint32) (*FourByteSignature, error) {
key := packFourByteKey(fourBytes, id)
val, err := d.db.GetCF(d.ro, d.cfh[cfFunctionSignatures], key)
if err != nil {
return nil, err
}
defer val.Free()
buf := val.Data()
if len(buf) == 0 {
return nil, nil
}
return unpackFourByteSignature(buf)
}
func (d *RocksDB) StoreFourByteSignature(wb *gorocksdb.WriteBatch, fourBytes uint32, id uint32, signature *FourByteSignature) error {
key := packFourByteKey(fourBytes, id)
wb.PutCF(d.cfh[cfFunctionSignatures], key, packFourByteSignature(signature))
return nil
}
func (d *RocksDB) GetEthereumInternalData(txid string) (*bchain.EthereumInternalData, error) {
btxID, err := d.chainParser.PackTxid(txid)
if err != nil {
@ -968,7 +1020,7 @@ func (d *RocksDB) DisconnectBlockRangeEthereumType(lower uint32, higher uint32)
wb.DeleteCF(d.cfh[cfBlockInternalDataErrors], key)
}
d.storeAddressContracts(wb, contracts)
err := d.db.Write(d.wo, wb)
err := d.WriteBatch(wb)
if err == nil {
d.is.RemoveLastBlockTimes(int(higher-lower) + 1)
glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher)

64
db/rocksdb_ethereumtype_test.go
View File

@ -8,6 +8,7 @@ import (
"reflect"
"testing"
"github.com/flier/gorocksdb"
"github.com/juju/errors"
"github.com/trezor/blockbook/bchain"
"github.com/trezor/blockbook/bchain/coins/eth"
@ -297,6 +298,30 @@ func formatInternalData(in *bchain.EthereumInternalData) *bchain.EthereumInterna
return &out
}
func testFourByteSignature(t *testing.T, d *RocksDB) {
fourBytes := uint32(1234123)
id := uint32(42313)
signature := FourByteSignature{
Name: "xyz",
Parameters: []string{"address", "(bytes,uint256[],uint256)", "uint16"},
}
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
if err := d.StoreFourByteSignature(wb, fourBytes, id, &signature); err != nil {
t.Fatal(err)
}
if err := d.WriteBatch(wb); err != nil {
t.Fatal(err)
}
got, err := d.GetFourByteSignature(fourBytes, id)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(*got, signature) {
t.Errorf("testFourByteSignature: got %+v, want %+v", got, signature)
}
}
// TestRocksDB_Index_EthereumType is an integration test probing the whole indexing functionality for EthereumType chains
// It does the following:
// 1) Connect two blocks (inputs from 2nd block are spending some outputs from the 1st block)
@ -417,6 +442,9 @@ func TestRocksDB_Index_EthereumType(t *testing.T) {
t.Errorf("GetBlockInfo() = %+v, want %+v", info, iw)
}
// Test to store and get FourByteSignature
testFourByteSignature(t, d)
// Test tx caching functionality, leave one tx in db to test cleanup in DisconnectBlock
testTxCache(t, d, block1, &block1.Txs[0])
// InternalData are not packed and stored in DB, remove them so that the test does not fail
@ -1133,3 +1161,39 @@ func Test_packUnpackBlockTx(t *testing.T) {
})
}
}
func Test_packUnpackFourByteSignature(t *testing.T) {
tests := []struct {
name string
signature FourByteSignature
}{
{
name: "no params",
signature: FourByteSignature{
Name: "abcdef",
},
},
{
name: "one param",
signature: FourByteSignature{
Name: "opqr",
Parameters: []string{"uint16"},
},
},
{
name: "multiple params",
signature: FourByteSignature{
Name: "xyz",
Parameters: []string{"address", "(bytes,uint256[],uint256)", "uint16"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buf := packFourByteSignature(&tt.signature)
if got, err := unpackFourByteSignature(buf); !reflect.DeepEqual(*got, tt.signature) || err != nil {
t.Errorf("packUnpackFourByteSignature() = %v, want %v, error %v", *got, tt.signature, err)
}
})
}
}

18
db/rocksdb_test.go
View File

@ -1481,3 +1481,21 @@ func Test_reorderUtxo(t *testing.T) {
})
}
}
func Test_packUnpackString(t *testing.T) {
tests := []struct {
name string
}{
{name: "ahoj"},
{name: ""},
{name: "very long long very long long very long long very long long very long long very long long very long long very long long very long long very long long very long long very long long very long long"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
buf := packString(tt.name)
if got, l := unpackString(buf); !reflect.DeepEqual(got, tt.name) || l != len(buf) {
t.Errorf("Test_packUnpackString() = %v, want %v, len %d, want len %d", got, tt.name, l, len(buf))
}
})
}
}

2
fiat/fiat_rates.go
View File

@ -30,7 +30,7 @@ type RatesDownloader struct {
downloader RatesDownloaderInterface
}
// NewFiatRatesDownloader initiallizes the downloader for FiatRates API.
// NewFiatRatesDownloader initializes the downloader for FiatRates API.
// If the startTime is nil, the downloader will start from the beginning.
func NewFiatRatesDownloader(db *db.RocksDB, apiType string, params string, startTime *time.Time, callback OnNewFiatRatesTicker) (*RatesDownloader, error) {
var rd = &RatesDownloader{}

199
fourbyte/fourbyte.go Normal file
View File

@ -0,0 +1,199 @@
package fourbyte
import (
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
"github.com/flier/gorocksdb"
"github.com/golang/glog"
"github.com/trezor/blockbook/db"
)
// Coingecko is a structure that implements RatesDownloaderInterface
type FourByteSignaturesDownloader struct {
url string
httpTimeoutSeconds time.Duration
db *db.RocksDB
}
// NewFourByteSignaturesDownloader initializes the downloader for FourByteSignatures API.
func NewFourByteSignaturesDownloader(db *db.RocksDB, url string) (*FourByteSignaturesDownloader, error) {
return &FourByteSignaturesDownloader{
url: url,
httpTimeoutSeconds: 15 * time.Second,
db: db,
}, nil
}
// Run starts the FourByteSignatures downloader
func (fd *FourByteSignaturesDownloader) Run() {
period := time.Hour * 24
timer := time.NewTimer(period)
for {
fd.downloadSignatures()
<-timer.C
timer.Reset(period)
}
}
type signatureData struct {
Id int `json:"id"`
TextSignature string `json:"text_signature"`
HexSignature string `json:"hex_signature"`
}
type signaturesPage struct {
Count int `json:"count"`
Next string `json:"next"`
Results []signatureData `json:"results"`
}
func (fd *FourByteSignaturesDownloader) getPage(url string) (*signaturesPage, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
glog.Errorf("Error creating a new request for %v: %v", url, err)
return nil, err
}
req.Close = true
req.Header.Set("Content-Type", "application/json")
client := &http.Client{
Timeout: fd.httpTimeoutSeconds,
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, errors.New("Invalid response status: " + string(resp.Status))
}
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var data signaturesPage
err = json.Unmarshal(bodyBytes, &data)
if err != nil {
glog.Errorf("Error parsing 4byte signatures response from %s: %v", url, err)
return nil, err
}
return &data, nil
}
func (fd *FourByteSignaturesDownloader) getPageWithRetry(url string) (*signaturesPage, error) {
for retry := 1; retry <= 16; retry++ {
page, err := fd.getPage(url)
if err == nil && page != nil {
return page, err
}
glog.Errorf("Error getting 4byte signatures from %s: %v, retry count %d", url, err, retry)
timer := time.NewTimer(time.Second * time.Duration(retry))
<-timer.C
}
return nil, errors.New("Too many retries to 4byte signatures")
}
func parseSignatureFromText(t string) *db.FourByteSignature {
s := strings.Index(t, "(")
e := strings.LastIndex(t, ")")
if s < 0 || e < 0 {
return nil
}
var signature db.FourByteSignature
signature.Name = t[:s]
params := t[s+1 : e]
if len(params) > 0 {
s = 0
tupleDepth := 0
// parse params as comma separated list
// tuple is regarded as one parameter and not parsed further
for i, c := range params {
if c == ',' && tupleDepth == 0 {
signature.Parameters = append(signature.Parameters, params[s:i])
s = i + 1
} else if c == '(' {
tupleDepth++
} else if c == ')' {
tupleDepth--
}
}
signature.Parameters = append(signature.Parameters, params[s:])
}
return &signature
}
func (fd *FourByteSignaturesDownloader) downloadSignatures() {
period := time.Millisecond * 100
timer := time.NewTimer(period)
url := fd.url
results := make([]signatureData, 0)
glog.Info("FourByteSignaturesDownloader starting download")
for {
page, err := fd.getPageWithRetry(url)
if err != nil {
glog.Errorf("Error getting 4byte signatures from %s: %v", url, err)
return
}
if page == nil {
glog.Errorf("Empty page from 4byte signatures from %s: %v", url, err)
return
}
glog.Infof("FourByteSignaturesDownloader downloaded %s with %d results", url, len(page.Results))
if len(page.Results) > 0 {
fourBytes, err := strconv.ParseUint(page.Results[0].HexSignature, 0, 0)
if err != nil {
glog.Errorf("Invalid 4byte signature %+v on page %s: %v", page.Results[0], url, err)
return
}
sig, err := fd.db.GetFourByteSignature(uint32(fourBytes), uint32(page.Results[0].Id))
if err != nil {
glog.Errorf("db.GetFourByteSignature error %+v on page %s: %v", page.Results[0], url, err)
return
}
// signature is already stored in db, break
if sig != nil {
break
}
results = append(results, page.Results...)
}
if page.Next == "" {
// at the end
break
}
url = page.Next
// wait a bit to not to flood the server
<-timer.C
timer.Reset(period)
}
if len(results) > 0 {
glog.Infof("FourByteSignaturesDownloader storing %d new signatures", len(results))
wb := gorocksdb.NewWriteBatch()
defer wb.Destroy()
for i := range results {
r := &results[i]
fourBytes, err := strconv.ParseUint(r.HexSignature, 0, 0)
if err != nil {
glog.Errorf("Invalid 4byte signature %+v: %v", r, err)
return
}
fbs := parseSignatureFromText(r.TextSignature)
if fbs != nil {
fd.db.StoreFourByteSignature(wb, uint32(fourBytes), uint32(r.Id), fbs)
} else {
glog.Errorf("FourByteSignaturesDownloader invalid signature %s", r.TextSignature)
}
}
if err := fd.db.WriteBatch(wb); err != nil {
glog.Errorf("FourByteSignaturesDownloader failed to store signatures, %v", err)
}
}
glog.Infof("FourByteSignaturesDownloader finished")
}

55
fourbyte/fourbyte_test.go Normal file
View File

@ -0,0 +1,55 @@
package fourbyte
import (
"reflect"
"testing"
"github.com/trezor/blockbook/db"
)
func Test_parseSignatureFromText(t *testing.T) {
tests := []struct {
name string
signature string
want db.FourByteSignature
}{
{
name: "_gonsPerFragment",
signature: "_gonsPerFragment()",
want: db.FourByteSignature{
Name: "_gonsPerFragment",
},
},
{
name: "vestingDeposits",
signature: "vestingDeposits(address)",
want: db.FourByteSignature{
Name: "vestingDeposits",
Parameters: []string{"address"},
},
},
{
name: "batchTransferTokenB",
signature: "batchTransferTokenB(address[],uint256)",
want: db.FourByteSignature{
Name: "batchTransferTokenB",
Parameters: []string{"address[]", "uint256"},
},
},
{
name: "transmitAndSellTokenForEth",
signature: "transmitAndSellTokenForEth(address,uint256,uint256,uint256,address,(uint8,bytes32,bytes32),bytes)",
want: db.FourByteSignature{
Name: "transmitAndSellTokenForEth",
Parameters: []string{"address", "uint256", "uint256", "uint256", "address", "(uint8,bytes32,bytes32)", "bytes"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := parseSignatureFromText(tt.signature); !reflect.DeepEqual(*got, tt.want) {
t.Errorf("parseSignatureFromText() = %v, want %v", *got, tt.want)
}
})
}
}