diff --git a/bchain/coins/btc/alternativefeeprovider.go b/bchain/coins/btc/alternativefeeprovider.go new file mode 100644 index 00000000..6bdf0e6a --- /dev/null +++ b/bchain/coins/btc/alternativefeeprovider.go @@ -0,0 +1,68 @@ +package btc + +import ( + "fmt" + "math/big" + "sync" + "time" + + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/trezor/blockbook/bchain" +) + +type alternativeFeeProviderFee struct { + blocks int + feePerKB int +} + +type alternativeFeeProvider struct { + fees []alternativeFeeProviderFee + lastSync time.Time + chain bchain.BlockChain + mux sync.Mutex +} + +type alternativeFeeProviderInterface interface { + compareToDefault() + estimateFee(blocks int) (big.Int, error) +} + +func (p *alternativeFeeProvider) compareToDefault() { + output := "" + for _, fee := range p.fees { + conservative, err := p.chain.(*BitcoinRPC).blockchainEstimateSmartFee(fee.blocks, true) + if err != nil { + glog.Error(err) + return + } + economical, err := p.chain.(*BitcoinRPC).blockchainEstimateSmartFee(fee.blocks, false) + if err != nil { + glog.Error(err) + return + } + output += fmt.Sprintf("Blocks %d: alternative %d, conservative %s, economical %s\n", fee.blocks, fee.feePerKB, conservative.String(), economical.String()) + } + glog.Info("alternativeFeeProviderCompareToDefault\n", output) +} + +func (p *alternativeFeeProvider) estimateFee(blocks int) (big.Int, error) { + var r big.Int + p.mux.Lock() + defer p.mux.Unlock() + if len(p.fees) == 0 { + return r, errors.New("alternativeFeeProvider: no fees") + } + if p.lastSync.Before(time.Now().Add(time.Duration(-10) * time.Minute)) { + return r, errors.Errorf("alternativeFeeProvider: Missing recent value, last sync at %v", p.lastSync) + } + for i := range p.fees { + if p.fees[i].blocks >= blocks { + r = *big.NewInt(int64(p.fees[i].feePerKB)) + return r, nil + } + } + // use the last value as fallback + r = *big.NewInt(int64(p.fees[len(p.fees)-1].feePerKB)) + return r, nil +} diff --git a/bchain/coins/btc/bitcoinrpc.go b/bchain/coins/btc/bitcoinrpc.go index f37d3772..510f821b 100644 --- a/bchain/coins/btc/bitcoinrpc.go +++ b/bchain/coins/btc/bitcoinrpc.go @@ -22,19 +22,20 @@ import ( // BitcoinRPC is an interface to JSON-RPC bitcoind service. type BitcoinRPC struct { *bchain.BaseChain - client http.Client - rpcURL string - user string - password string - Mempool *bchain.MempoolBitcoinType - ParseBlocks bool - pushHandler func(bchain.NotificationType) - mq *bchain.MQ - ChainConfig *Configuration - RPCMarshaler RPCMarshaler - mempoolGolombFilterP uint8 - mempoolFilterScripts string - mempoolUseZeroedKey bool + client http.Client + rpcURL string + user string + password string + Mempool *bchain.MempoolBitcoinType + ParseBlocks bool + pushHandler func(bchain.NotificationType) + mq *bchain.MQ + ChainConfig *Configuration + RPCMarshaler RPCMarshaler + mempoolGolombFilterP uint8 + mempoolFilterScripts string + mempoolUseZeroedKey bool + alternativeFeeProvider alternativeFeeProviderInterface } // Configuration represents json config file @@ -145,10 +146,16 @@ func (b *BitcoinRPC) Initialize() error { glog.Info("rpc: block chain ", params.Name) if b.ChainConfig.AlternativeEstimateFee == "whatthefee" { - if err = InitWhatTheFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil { - glog.Error("InitWhatTheFee error ", err, " Reverting to default estimateFee functionality") + if b.alternativeFeeProvider, err = NewWhatTheFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil { + glog.Error("NewWhatTheFee error ", err, " Reverting to default estimateFee functionality") // disable AlternativeEstimateFee logic - b.ChainConfig.AlternativeEstimateFee = "" + b.alternativeFeeProvider = nil + } + } else if b.ChainConfig.AlternativeEstimateFee == "mempoolspace" { + if b.alternativeFeeProvider, err = NewMempoolSpaceFee(b, b.ChainConfig.AlternativeEstimateFeeParams); err != nil { + glog.Error("MempoolSpaceFee error ", err, " Reverting to default estimateFee functionality") + // disable AlternativeEstimateFee logic + b.alternativeFeeProvider = nil } } @@ -774,8 +781,7 @@ func (b *BitcoinRPC) getRawTransaction(txid string) (json.RawMessage, error) { return res.Result, nil } -// EstimateSmartFee returns fee estimation -func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { +func (b *BitcoinRPC) blockchainEstimateSmartFee(blocks int, conservative bool) (big.Int, error) { // use EstimateFee if EstimateSmartFee is not supported if !b.ChainConfig.SupportsEstimateSmartFee && b.ChainConfig.SupportsEstimateFee { return b.EstimateFee(blocks) @@ -792,7 +798,6 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, e req.Params.EstimateMode = "ECONOMICAL" } err := b.Call(&req, &res) - var r big.Int if err != nil { return r, err @@ -807,8 +812,31 @@ func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, e return r, nil } +// EstimateSmartFee returns fee estimation +func (b *BitcoinRPC) EstimateSmartFee(blocks int, conservative bool) (big.Int, error) { + // use alternative estimator if enabled + if b.alternativeFeeProvider != nil { + r, err := b.alternativeFeeProvider.estimateFee(blocks) + // in case of error, fallback to default estimator + if err == nil { + return r, nil + } + } + return b.blockchainEstimateSmartFee(blocks, conservative) +} + // EstimateFee returns fee estimation. func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { + var r big.Int + var err error + // use alternative estimator if enabled + if b.alternativeFeeProvider != nil { + r, err = b.alternativeFeeProvider.estimateFee(blocks) + // in case of error, fallback to default estimator + if err == nil { + return r, nil + } + } // use EstimateSmartFee if EstimateFee is not supported if !b.ChainConfig.SupportsEstimateFee && b.ChainConfig.SupportsEstimateSmartFee { return b.EstimateSmartFee(blocks, true) @@ -819,9 +847,8 @@ func (b *BitcoinRPC) EstimateFee(blocks int) (big.Int, error) { res := ResEstimateFee{} req := CmdEstimateFee{Method: "estimatefee"} req.Params.Blocks = blocks - err := b.Call(&req, &res) + err = b.Call(&req, &res) - var r big.Int if err != nil { return r, err } diff --git a/bchain/coins/btc/mempoolspace.go b/bchain/coins/btc/mempoolspace.go new file mode 100644 index 00000000..01eb9a40 --- /dev/null +++ b/bchain/coins/btc/mempoolspace.go @@ -0,0 +1,135 @@ +package btc + +import ( + "bytes" + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/golang/glog" + "github.com/juju/errors" + "github.com/trezor/blockbook/bchain" +) + +// https://mempool.space/api/v1/fees/recommended returns +// {"fastestFee":41,"halfHourFee":39,"hourFee":36,"economyFee":36,"minimumFee":20} + +type mempoolSpaceFeeResult struct { + FastestFee int `json:"fastestFee"` + HalfHourFee int `json:"halfHourFee"` + HourFee int `json:"hourFee"` + EconomyFee int `json:"economyFee"` + MinimumFee int `json:"minimumFee"` +} + +type mempoolSpaceFeeParams struct { + URL string `json:"url"` + PeriodSeconds int `periodSeconds:"url"` +} + +type mempoolSpaceFeeProvider struct { + *alternativeFeeProvider + params mempoolSpaceFeeParams +} + +// NewMempoolSpaceFee initializes https://mempool.space provider +func NewMempoolSpaceFee(chain bchain.BlockChain, params string) (alternativeFeeProviderInterface, error) { + p := &mempoolSpaceFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}} + err := json.Unmarshal([]byte(params), &p.params) + if err != nil { + return nil, err + } + if p.params.URL == "" || p.params.PeriodSeconds == 0 { + return nil, errors.New("NewWhatTheFee: Missing parameters") + } + p.chain = chain + go p.mempoolSpaceFeeDownloader() + return p, nil +} + +func (p *mempoolSpaceFeeProvider) mempoolSpaceFeeDownloader() { + period := time.Duration(p.params.PeriodSeconds) * time.Second + timer := time.NewTimer(period) + counter := 0 + for { + var data mempoolSpaceFeeResult + err := p.mempoolSpaceFeeGetData(&data) + if err != nil { + glog.Error("mempoolSpaceFeeGetData ", err) + } else { + if p.mempoolSpaceFeeProcessData(&data) { + if counter%60 == 0 { + p.compareToDefault() + } + counter++ + } + } + <-timer.C + timer.Reset(period) + } +} + +func (p *mempoolSpaceFeeProvider) mempoolSpaceFeeProcessData(data *mempoolSpaceFeeResult) bool { + if data.MinimumFee == 0 || data.EconomyFee == 0 || data.HourFee == 0 || data.HalfHourFee == 0 || data.FastestFee == 0 { + glog.Errorf("mempoolSpaceFeeProcessData: invalid data %+v", data) + return false + } + p.mux.Lock() + defer p.mux.Unlock() + p.fees = make([]alternativeFeeProviderFee, 5) + // map mempoool.space fees to blocks + + // FastestFee is for 1 block + p.fees[0] = alternativeFeeProviderFee{ + blocks: 1, + feePerKB: data.FastestFee * 1000, + } + + // HalfHourFee is for 2-5 blocks + p.fees[1] = alternativeFeeProviderFee{ + blocks: 5, + feePerKB: data.HalfHourFee * 1000, + } + + // HourFee is for 6-18 blocks + p.fees[2] = alternativeFeeProviderFee{ + blocks: 18, + feePerKB: data.HourFee * 1000, + } + + // EconomyFee is for 19-100 blocks + p.fees[3] = alternativeFeeProviderFee{ + blocks: 100, + feePerKB: data.EconomyFee * 1000, + } + + // MinimumFee is for over 100 blocks + p.fees[4] = alternativeFeeProviderFee{ + blocks: 500, + feePerKB: data.MinimumFee * 1000, + } + + p.lastSync = time.Now() + // glog.Infof("mempoolSpaceFees: %+v", p.fees) + return true +} + +func (p *mempoolSpaceFeeProvider) mempoolSpaceFeeGetData(res interface{}) error { + var httpData []byte + httpReq, err := http.NewRequest("GET", p.params.URL, bytes.NewBuffer(httpData)) + if err != nil { + return err + } + httpRes, err := http.DefaultClient.Do(httpReq) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return err + } + if httpRes.StatusCode != http.StatusOK { + return errors.New(p.params.URL + " returned status " + strconv.Itoa(httpRes.StatusCode)) + } + return safeDecodeResponse(httpRes.Body, &res) +} diff --git a/bchain/coins/btc/mempoolspace_test.go b/bchain/coins/btc/mempoolspace_test.go new file mode 100644 index 00000000..ba4421cd --- /dev/null +++ b/bchain/coins/btc/mempoolspace_test.go @@ -0,0 +1,47 @@ +package btc + +import ( + "math/big" + "strconv" + "testing" +) + +func Test_mempoolSpaceFeeProvider(t *testing.T) { + m := &mempoolSpaceFeeProvider{alternativeFeeProvider: &alternativeFeeProvider{}} + m.mempoolSpaceFeeProcessData(&mempoolSpaceFeeResult{ + MinimumFee: 10, + EconomyFee: 20, + HourFee: 30, + HalfHourFee: 40, + FastestFee: 50, + }) + + tests := []struct { + blocks int + want big.Int + }{ + {0, *big.NewInt(50000)}, + {1, *big.NewInt(50000)}, + {2, *big.NewInt(40000)}, + {5, *big.NewInt(40000)}, + {6, *big.NewInt(30000)}, + {10, *big.NewInt(30000)}, + {18, *big.NewInt(30000)}, + {19, *big.NewInt(20000)}, + {100, *big.NewInt(20000)}, + {101, *big.NewInt(10000)}, + {500, *big.NewInt(10000)}, + {5000000, *big.NewInt(10000)}, + } + for _, tt := range tests { + t.Run(strconv.Itoa(tt.blocks), func(t *testing.T) { + got, err := m.estimateFee(tt.blocks) + if err != nil { + t.Error("estimateFee returned error ", err) + } + if got.Cmp(&tt.want) != 0 { + t.Errorf("estimateFee(%d) = %v, want %v", tt.blocks, got, tt.want) + } + }) + } +} diff --git a/bchain/coins/btc/whatthefee.go b/bchain/coins/btc/whatthefee.go index c0977f80..c7567f7a 100644 --- a/bchain/coins/btc/whatthefee.go +++ b/bchain/coins/btc/whatthefee.go @@ -3,11 +3,9 @@ package btc import ( "bytes" "encoding/json" - "fmt" "math" "net/http" "strconv" - "sync" "time" "github.com/golang/glog" @@ -34,49 +32,40 @@ type whatTheFeeParams struct { PeriodSeconds int `periodSeconds:"url"` } -type whatTheFeeFee struct { - blocks int - feesPerKB []int -} - -type whatTheFeeData struct { +type whatTheFeeProvider struct { + *alternativeFeeProvider params whatTheFeeParams probabilities []string - fees []whatTheFeeFee - lastSync time.Time - chain bchain.BlockChain - mux sync.Mutex } -var whatTheFee whatTheFeeData - -// InitWhatTheFee initializes https://whatthefee.io handler -func InitWhatTheFee(chain bchain.BlockChain, params string) error { - err := json.Unmarshal([]byte(params), &whatTheFee.params) +// NewWhatTheFee initializes https://whatthefee.io provider +func NewWhatTheFee(chain bchain.BlockChain, params string) (alternativeFeeProviderInterface, error) { + var p whatTheFeeProvider + err := json.Unmarshal([]byte(params), &p.params) if err != nil { - return err + return nil, err } - if whatTheFee.params.URL == "" || whatTheFee.params.PeriodSeconds == 0 { - return errors.New("Missing parameters") + if p.params.URL == "" || p.params.PeriodSeconds == 0 { + return nil, errors.New("NewWhatTheFee: Missing parameters") } - whatTheFee.chain = chain - go whatTheFeeDownloader() - return nil + p.chain = chain + go p.whatTheFeeDownloader() + return &p, nil } -func whatTheFeeDownloader() { - period := time.Duration(whatTheFee.params.PeriodSeconds) * time.Second +func (p *whatTheFeeProvider) whatTheFeeDownloader() { + period := time.Duration(p.params.PeriodSeconds) * time.Second timer := time.NewTimer(period) counter := 0 for { var data whatTheFeeServiceResult - err := whatTheFeeGetData(&data) + err := p.whatTheFeeGetData(&data) if err != nil { glog.Error("whatTheFeeGetData ", err) } else { - if whatTheFeeProcessData(&data) { + if p.whatTheFeeProcessData(&data) { if counter%60 == 0 { - whatTheFeeCompareToDefault() + p.compareToDefault() } counter++ } @@ -86,15 +75,15 @@ func whatTheFeeDownloader() { } } -func whatTheFeeProcessData(data *whatTheFeeServiceResult) bool { +func (p *whatTheFeeProvider) whatTheFeeProcessData(data *whatTheFeeServiceResult) bool { if len(data.Index) == 0 || len(data.Index) != len(data.Data) || len(data.Columns) == 0 { glog.Errorf("invalid data %+v", data) return false } - whatTheFee.mux.Lock() - defer whatTheFee.mux.Unlock() - whatTheFee.probabilities = data.Columns - whatTheFee.fees = make([]whatTheFeeFee, len(data.Index)) + p.mux.Lock() + defer p.mux.Unlock() + p.probabilities = data.Columns + p.fees = make([]alternativeFeeProviderFee, len(data.Index)) for i, blocks := range data.Index { if len(data.Columns) != len(data.Data[i]) { glog.Errorf("invalid data %+v", data) @@ -104,19 +93,19 @@ func whatTheFeeProcessData(data *whatTheFeeServiceResult) bool { for j, l := range data.Data[i] { fees[j] = int(1000 * math.Exp(float64(l)/100)) } - whatTheFee.fees[i] = whatTheFeeFee{ - blocks: blocks, - feesPerKB: fees, + p.fees[i] = alternativeFeeProviderFee{ + blocks: blocks, + feePerKB: fees[len(fees)/2], } } - whatTheFee.lastSync = time.Now() - glog.Infof("%+v", whatTheFee.fees) + p.lastSync = time.Now() + glog.Infof("whatTheFees: %+v", p.fees) return true } -func whatTheFeeGetData(res interface{}) error { +func (p *whatTheFeeProvider) whatTheFeeGetData(res interface{}) error { var httpData []byte - httpReq, err := http.NewRequest("GET", whatTheFee.params.URL, bytes.NewBuffer(httpData)) + httpReq, err := http.NewRequest("GET", p.params.URL, bytes.NewBuffer(httpData)) if err != nil { return err } @@ -132,25 +121,3 @@ func whatTheFeeGetData(res interface{}) error { } return safeDecodeResponse(httpRes.Body, &res) } - -func whatTheFeeCompareToDefault() { - output := "" - for _, fee := range whatTheFee.fees { - output += fmt.Sprint(fee.blocks, ",") - for _, wtf := range fee.feesPerKB { - output += fmt.Sprint(wtf, ",") - } - conservative, err := whatTheFee.chain.EstimateSmartFee(fee.blocks, true) - if err != nil { - glog.Error(err) - return - } - economical, err := whatTheFee.chain.EstimateSmartFee(fee.blocks, false) - if err != nil { - glog.Error(err) - return - } - output += fmt.Sprint(conservative.String(), ",", economical.String(), "\n") - } - glog.Info("whatTheFeeCompareToDefault\n", output) -} diff --git a/configs/coins/bitcoin.json b/configs/coins/bitcoin.json index e6a4656c..2f4a7ac2 100644 --- a/configs/coins/bitcoin.json +++ b/configs/coins/bitcoin.json @@ -64,8 +64,8 @@ "xpub_magic_segwit_p2sh": 77429938, "xpub_magic_segwit_native": 78792518, "additional_params": { - "alternative_estimate_fee": "whatthefee-disabled", - "alternative_estimate_fee_params": "{\"url\": \"https://whatthefee.io/data.json\", \"periodSeconds\": 60}", + "alternative_estimate_fee": "mempoolspace", + "alternative_estimate_fee_params": "{\"url\": \"https://mempool.space/api/v1/fees/recommended\", \"periodSeconds\": 20}", "fiat_rates": "coingecko", "fiat_rates_vs_currencies": "AED,ARS,AUD,BDT,BHD,BMD,BRL,CAD,CHF,CLP,CNY,CZK,DKK,EUR,GBP,HKD,HUF,IDR,ILS,INR,JPY,KRW,KWD,LKR,MMK,MXN,MYR,NGN,NOK,NZD,PHP,PKR,PLN,RUB,SAR,SEK,SGD,THB,TRY,TWD,UAH,USD,VEF,VND,ZAR,BTC,ETH", "fiat_rates_params": "{\"coin\": \"bitcoin\", \"periodSeconds\": 900}",