diff --git a/db/fiat.go b/db/fiat.go index 040494b3..dee8aa94 100644 --- a/db/fiat.go +++ b/db/fiat.go @@ -84,20 +84,6 @@ func unpackCurrencyRatesTicker(buf []byte) (*common.CurrencyRatesTicker, error) return &ticker, nil } -// FiatRatesConvertDate checks if the date is in correct format and returns the Time object. -// Possible formats are: YYYYMMDDhhmmss, YYYYMMDDhhmm, YYYYMMDDhh, YYYYMMDD -func FiatRatesConvertDate(date string) (*time.Time, error) { - for format := FiatRatesTimeFormat; len(format) >= 8; format = format[:len(format)-2] { - convertedDate, err := time.Parse(format, date) - if err == nil { - return &convertedDate, nil - } - } - msg := "Date \"" + date + "\" does not match any of available formats. " - msg += "Possible formats are: YYYYMMDDhhmmss, YYYYMMDDhhmm, YYYYMMDDhh, YYYYMMDD" - return nil, errors.New(msg) -} - // FiatRatesStoreTicker stores ticker data at the specified time func (d *RocksDB) FiatRatesStoreTicker(wb *grocksdb.WriteBatch, ticker *common.CurrencyRatesTicker) error { if len(ticker.Rates) == 0 { @@ -145,22 +131,6 @@ func (d *RocksDB) FiatRatesGetTicker(tickerTime *time.Time) (*common.CurrencyRat // FiatRatesFindTicker gets FiatRates data closest to the specified timestamp, of the base currency, vsCurrency or the token if specified func (d *RocksDB) FiatRatesFindTicker(tickerTime *time.Time, vsCurrency string, token string) (*common.CurrencyRatesTicker, error) { - // currentTicker := d.is.GetCurrentTicker("", "") - // lastTickerInDBMux.Lock() - // dbTicker := lastTickerInDB - // lastTickerInDBMux.Unlock() - // if currentTicker != nil { - // if !tickerTime.Before(currentTicker.Timestamp) || (dbTicker != nil && tickerTime.After(dbTicker.Timestamp)) { - // f := true - // if token != "" && currentTicker.TokenRates != nil { - // _, f = currentTicker.TokenRates[token] - // } - // if f { - // return currentTicker, nil - // } - // } - // } - tickerTimeFormatted := tickerTime.UTC().Format(FiatRatesTimeFormat) it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates]) defer it.Close() @@ -178,6 +148,26 @@ func (d *RocksDB) FiatRatesFindTicker(tickerTime *time.Time, vsCurrency string, return nil, nil } +// FiatRatesGetAllTickers gets FiatRates data closest to the specified timestamp, of the base currency, vsCurrency or the token if specified +func (d *RocksDB) FiatRatesGetAllTickers(fn func(ticker *common.CurrencyRatesTicker) error) error { + it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates]) + defer it.Close() + + for it.SeekToFirst(); it.Valid(); it.Next() { + ticker, err := getTickerFromIterator(it, "", "") + if err != nil { + return err + } + if ticker == nil { + return errors.New("FiatRatesGetAllTickers got nil ticker") + } + if err = fn(ticker); err != nil { + return err + } + } + return nil +} + // FiatRatesFindLastTicker gets the last FiatRates record, of the base currency, vsCurrency or the token if specified func (d *RocksDB) FiatRatesFindLastTicker(vsCurrency string, token string) (*common.CurrencyRatesTicker, error) { it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates]) diff --git a/db/fiat_test.go b/db/fiat_test.go index 05f3ee54..e9ce5b4e 100644 --- a/db/fiat_test.go +++ b/db/fiat_test.go @@ -17,22 +17,6 @@ func TestRocksTickers(t *testing.T) { }) defer closeAndDestroyRocksDB(t, d) - // Test valid formats - for _, date := range []string{"20190130", "2019013012", "201901301250", "20190130125030"} { - _, err := FiatRatesConvertDate(date) - if err != nil { - t.Errorf("%v", err) - } - } - - // Test invalid formats - for _, date := range []string{"01102019", "10201901", "", "abc", "20190130xxx"} { - _, err := FiatRatesConvertDate(date) - if err == nil { - t.Errorf("Wrongly-formatted date \"%v\" marked as valid!", date) - } - } - // Test storing & finding tickers pastKey, _ := time.Parse(FiatRatesTimeFormat, "20190627000000") futureKey, _ := time.Parse(FiatRatesTimeFormat, "20190630000000") diff --git a/fiat/coingecko.go b/fiat/coingecko.go index 418a2db2..d4dcef9b 100644 --- a/fiat/coingecko.go +++ b/fiat/coingecko.go @@ -3,7 +3,7 @@ package fiat import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strconv" @@ -86,7 +86,7 @@ func doReq(req *http.Request, client *http.Client) ([]byte, error) { return nil, err } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -299,14 +299,43 @@ func (cg *Coingecko) CurrentTickers() (*common.CurrencyRatesTicker, error) { return &newTickers, nil } +func (cg *Coingecko) getHighGranularityTickers(days string) (*[]common.CurrencyRatesTicker, error) { + mc, err := cg.coinMarketChart(cg.coin, highGranularityVsCurrency, days, false) + if err != nil { + return nil, err + } + if len(mc.Prices) < 2 { + return nil, nil + } + // ignore the last point, it is not in granularity + tickers := make([]common.CurrencyRatesTicker, len(mc.Prices)-1) + for i, p := range mc.Prices[:len(mc.Prices)-1] { + var timestamp uint + timestamp = uint(p[0]) + if timestamp > 100000000000 { + // convert timestamp from milliseconds to seconds + timestamp /= 1000 + } + rate := float32(p[1]) + u := time.Unix(int64(timestamp), 0).UTC() + ticker := common.CurrencyRatesTicker{ + Timestamp: u, + Rates: make(map[string]float32), + } + ticker.Rates[highGranularityVsCurrency] = rate + tickers[i] = ticker + } + return &tickers, nil +} + // HourlyTickers returns the array of the exchange rates in hourly granularity func (cg *Coingecko) HourlyTickers() (*[]common.CurrencyRatesTicker, error) { - return nil, nil + return cg.getHighGranularityTickers("90") } // HourlyTickers returns the array of the exchange rates in five minutes granularity func (cg *Coingecko) FiveMinutesTickers() (*[]common.CurrencyRatesTicker, error) { - return nil, nil + return cg.getHighGranularityTickers("1") } func (cg *Coingecko) getHistoricalTicker(tickersToUpdate map[uint]*common.CurrencyRatesTicker, coinId string, vsCurrency string, token string) (bool, error) { diff --git a/fiat/fiat_rates.go b/fiat/fiat_rates.go index b21699ec..b1f59ba8 100644 --- a/fiat/fiat_rates.go +++ b/fiat/fiat_rates.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "math/rand" + "os" "strings" "sync" "time" @@ -15,14 +15,20 @@ import ( "github.com/trezor/blockbook/db" ) -const CurrentTickersKey = "CurrentTickers" -const HourlyTickersKey = "HourlyTickers" -const FiveMinutesTickersKey = "FiveMinutesTickers" +const currentTickersKey = "CurrentTickers" +const hourlyTickersKey = "HourlyTickers" +const fiveMinutesTickersKey = "FiveMinutesTickers" + +const highGranularityVsCurrency = "usd" + +const secondsInDay = 24 * 60 * 60 +const secondsInHour = 60 * 60 +const secondsInFiveMinutes = 5 * 60 // OnNewFiatRatesTicker is used to send notification about a new FiatRates ticker type OnNewFiatRatesTicker func(ticker *common.CurrencyRatesTicker) -// RatesDownloaderInterface provides method signatures for specific fiat rates downloaders +// RatesDownloaderInterface provides method signatures for a specific fiat rates downloader type RatesDownloaderInterface interface { CurrentTickers() (*common.CurrencyRatesTicker, error) HourlyTickers() (*[]common.CurrencyRatesTicker, error) @@ -55,17 +61,6 @@ type FiatRates struct { dailyTickersTo int64 } -func tickersToMap(tickers *[]common.CurrencyRatesTicker, granularitySeconds int64) (map[int64]*common.CurrencyRatesTicker, int64, int64) { - if tickers == nil || len(*tickers) == 0 { - return nil, 0, 0 - } - halfGranularity := granularitySeconds / 2 - m := make(map[int64]*common.CurrencyRatesTicker, len(*tickers)) - from := ((*tickers)[0].Timestamp.UTC().Unix() + halfGranularity) % granularitySeconds - to := ((*tickers)[len(*tickers)-1].Timestamp.UTC().Unix() + halfGranularity) % granularitySeconds - return m, from, to -} - // NewFiatRates initializes the FiatRates handler func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTicker) (*FiatRates, error) { var config struct { @@ -73,7 +68,7 @@ func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTick FiatRatesParams string `json:"fiat_rates_params"` FiatRatesVsCurrencies string `json:"fiat_rates_vs_currencies"` } - data, err := ioutil.ReadFile(configFile) + data, err := os.ReadFile(configFile) if err != nil { return nil, fmt.Errorf("error reading file %v, %v", configFile, err) } @@ -133,7 +128,11 @@ func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTick is.HasTokenFiatRates = fr.downloadTokens fr.Enabled = true - currentTickers, err := db.FiatRatesGetSpecialTickers(CurrentTickersKey) + if err := fr.loadDailyTickers(); err != nil { + return nil, err + } + + currentTickers, err := db.FiatRatesGetSpecialTickers(currentTickersKey) if err != nil { glog.Error("FiatRatesDownloader: get CurrentTickers from DB error ", err) } @@ -141,22 +140,23 @@ func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTick fr.currentTicker = &(*currentTickers)[0] } - hourlyTickers, err := db.FiatRatesGetSpecialTickers(HourlyTickersKey) + hourlyTickers, err := db.FiatRatesGetSpecialTickers(hourlyTickersKey) if err != nil { glog.Error("FiatRatesDownloader: get HourlyTickers from DB error ", err) } - fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = tickersToMap(hourlyTickers, 3600) + fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = fr.tickersToMap(hourlyTickers, secondsInHour) - fiveMinutesTickers, err := db.FiatRatesGetSpecialTickers(FiveMinutesTickersKey) + fiveMinutesTickers, err := db.FiatRatesGetSpecialTickers(fiveMinutesTickersKey) if err != nil { glog.Error("FiatRatesDownloader: get FiveMinutesTickers from DB error ", err) } - fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = tickersToMap(fiveMinutesTickers, 5*60) + fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = fr.tickersToMap(fiveMinutesTickers, secondsInFiveMinutes) } } else { return nil, fmt.Errorf("unknown provider %q", fr.provider) } + fr.logTickersInfo() return fr, nil } @@ -171,28 +171,208 @@ func (fr *FiatRates) GetCurrentTicker(vsCurrency string, token string) *common.C return nil } +// getTokenTickersForTimestamps returns tickers for slice of timestamps, that contain requested vsCurrency and token +func (fr *FiatRates) getTokenTickersForTimestamps(timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) { + currentTicker := fr.GetCurrentTicker("", token) + tickers := make([]*common.CurrencyRatesTicker, len(timestamps)) + var prevTicker *common.CurrencyRatesTicker + for i, t := range timestamps { + // check if the token is available in the current ticker - if not, return nil ticker instead of wasting time in costly DB searches + if currentTicker != nil { + var ticker *common.CurrencyRatesTicker + date := time.Unix(t, 0) + // if previously found ticker is newer than this one (token tickers may not be in DB for every day), skip search in DB + if prevTicker != nil && !date.After(prevTicker.Timestamp) { + ticker = prevTicker + } else { + ticker, _ := fr.db.FiatRatesFindTicker(&date, vsCurrency, token) + prevTicker = ticker + } + // if ticker not found in DB, use current ticker + if ticker == nil { + tickers[i] = currentTicker + prevTicker = currentTicker + } else { + tickers[i] = ticker + } + } + } + return &tickers, nil +} + +// GetTickersForTimestamps returns tickers for slice of timestamps, that contain requested vsCurrency and token +func (fr *FiatRates) GetTickersForTimestamps(timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) { + // token rates are not in memory, them load from DB + if token != "" { + return fr.getTokenTickersForTimestamps(timestamps, vsCurrency, token) + } + fr.mux.RLock() + defer fr.mux.RUnlock() + tickers := make([]*common.CurrencyRatesTicker, len(timestamps)) + var prevTicker *common.CurrencyRatesTicker + for i, t := range timestamps { + dailyTs := normalizedUnix(t, secondsInDay) + // use higher granularity only for non daily timestamps + if t != dailyTs { + if t >= fr.fiveMinutesTickersFrom && t <= fr.fiveMinutesTickersTo { + if ticker, found := fr.fiveMinutesTickers[normalizedUnix(t, secondsInFiveMinutes)]; found && ticker != nil { + if common.IsSuitableTicker(ticker, vsCurrency, token) { + tickers[i] = ticker + continue + } + } + } + if t >= fr.hourlyTickersFrom && t <= fr.hourlyTickersTo { + if ticker, found := fr.hourlyTickers[normalizedUnix(t, secondsInHour)]; found && ticker != nil { + if common.IsSuitableTicker(ticker, vsCurrency, token) { + tickers[i] = ticker + continue + } + } + } + } + if prevTicker != nil && dailyTs >= prevTicker.Timestamp.Unix() { + tickers[i] = prevTicker + continue + } else { + var found bool + if dailyTs < fr.dailyTickersFrom { + dailyTs = fr.dailyTickersFrom + } + var ticker *common.CurrencyRatesTicker + for ; dailyTs <= fr.dailyTickersTo; dailyTs += secondsInDay { + if ticker, found = fr.dailyTickers[dailyTs]; found && ticker != nil { + if common.IsSuitableTicker(ticker, vsCurrency, token) { + tickers[i] = ticker + prevTicker = ticker + break + } else { + found = false + } + } + if !found { + tickers[i] = fr.currentTicker + prevTicker = fr.currentTicker + } + } + } + } + return &tickers, nil +} +func (fr *FiatRates) logTickersInfo() { + glog.Infof("fiat rates %s handler, %d (%s - %s) daily tickers, %d (%s - %s) hourly tickers, %d (%s - %s) 5 minute tickers", fr.provider, + len(fr.dailyTickers), time.Unix(fr.dailyTickersFrom, 0).Format("2006-01-02"), time.Unix(fr.dailyTickersTo, 0).Format("2006-01-02"), + len(fr.hourlyTickers), time.Unix(fr.hourlyTickersFrom, 0).Format("2006-01-02 15:04"), time.Unix(fr.hourlyTickersTo, 0).Format("2006-01-02 15:04"), + len(fr.fiveMinutesTickers), time.Unix(fr.fiveMinutesTickersFrom, 0).Format("2006-01-02 15:04"), time.Unix(fr.fiveMinutesTickersTo, 0).Format("2006-01-02 15:04")) +} + +func normalizedTimeUnix(t time.Time, granularity int64) int64 { + return normalizedUnix(t.UTC().Unix(), granularity) +} + +func normalizedUnix(t int64, granularity int64) int64 { + unix := t + (granularity >> 1) + return unix - unix%granularity +} + +// loadDailyTickers loads daily tickers to cache +func (fr *FiatRates) loadDailyTickers() error { + fr.mux.Lock() + defer fr.mux.Unlock() + fr.dailyTickers = make(map[int64]*common.CurrencyRatesTicker) + err := fr.db.FiatRatesGetAllTickers(func(ticker *common.CurrencyRatesTicker) error { + normalizedTime := normalizedTimeUnix(ticker.Timestamp, secondsInDay) + // remove token rates from cache to save memory (tickers with token rates are hundreds of kb big) + ticker.TokenRates = nil + if len(fr.dailyTickers) > 0 { + // check that there is a ticker for every day, if missing, set it from current value if missing + prevTime := normalizedTime + for { + prevTime -= secondsInDay + if _, found := fr.dailyTickers[prevTime]; found { + break + } + fr.dailyTickers[prevTime] = ticker + } + } else { + fr.dailyTickersFrom = normalizedTime + } + fr.dailyTickers[normalizedTime] = ticker + fr.dailyTickersTo = normalizedTime + return nil + }) + return err +} + // setCurrentTicker sets current ticker func (fr *FiatRates) setCurrentTicker(t *common.CurrencyRatesTicker) { fr.mux.Lock() defer fr.mux.Unlock() fr.currentTicker = t - fr.db.FiatRatesStoreSpecialTickers(CurrentTickersKey, &[]common.CurrencyRatesTicker{*t}) + fr.db.FiatRatesStoreSpecialTickers(currentTickersKey, &[]common.CurrencyRatesTicker{*t}) +} + +func (fr *FiatRates) tickersToMap(tickers *[]common.CurrencyRatesTicker, granularitySeconds int64) (map[int64]*common.CurrencyRatesTicker, int64, int64) { + if tickers == nil || len(*tickers) == 0 { + return make(map[int64]*common.CurrencyRatesTicker), 0, 0 + } + m := make(map[int64]*common.CurrencyRatesTicker, len(*tickers)) + from := int64(0) + to := int64(0) + for i := range *tickers { + ticker := (*tickers)[i] + normalizedTime := normalizedTimeUnix(ticker.Timestamp, granularitySeconds) + dailyTime := normalizedTimeUnix(ticker.Timestamp, secondsInDay) + dailyTicker, found := fr.dailyTickers[dailyTime] + if !found { + // if not found in historical tickers, use current ticker + dailyTicker = fr.currentTicker + } + if dailyTicker != nil { + // high granularity tickers are loaded only in one currency, add other currencies based on daily rate between fiat currencies + vsRate, foundVs := ticker.Rates[highGranularityVsCurrency] + dailyVsRate, foundDaily := dailyTicker.Rates[highGranularityVsCurrency] + if foundDaily && dailyVsRate != 0 && foundVs && vsRate != 0 { + for currency, rate := range dailyTicker.Rates { + if currency != highGranularityVsCurrency { + ticker.Rates[currency] = vsRate * rate / dailyVsRate + } + } + } + } + if len(m) > 0 { + // check that there is a ticker for each period, set it from current value if missing + prevTime := normalizedTime + for { + prevTime -= granularitySeconds + if _, found := m[prevTime]; found { + break + } + m[prevTime] = &ticker + } + } else { + from = normalizedTime + } + m[normalizedTime] = &ticker + to = normalizedTime + } + return m, from, to } // setCurrentTicker sets hourly tickers func (fr *FiatRates) setHourlyTickers(t *[]common.CurrencyRatesTicker) { + fr.db.FiatRatesStoreSpecialTickers(hourlyTickersKey, t) fr.mux.Lock() defer fr.mux.Unlock() - fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = tickersToMap(t, 3600) - fr.db.FiatRatesStoreSpecialTickers(HourlyTickersKey, t) + fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = fr.tickersToMap(t, secondsInHour) } // setCurrentTicker sets hourly tickers func (fr *FiatRates) setFiveMinutesTickers(t *[]common.CurrencyRatesTicker) { + fr.db.FiatRatesStoreSpecialTickers(fiveMinutesTickersKey, t) fr.mux.Lock() defer fr.mux.Unlock() - fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = tickersToMap(t, 5*60) - fr.db.FiatRatesStoreSpecialTickers(FiveMinutesTickersKey, t) + fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = fr.tickersToMap(t, secondsInFiveMinutes) } // RunDownloader periodically downloads current (every 15 minutes) and historical (once a day) tickers @@ -245,13 +425,18 @@ func (fr *FiatRates) RunDownloader() error { glog.Error("FiatRatesDownloader: UpdateHistoricalTickers error ", err) } else { lastHistoricalTickers = time.Now().UTC() - ticker, err := fr.db.FiatRatesFindLastTicker("", "") - if err != nil || ticker == nil { - glog.Error("FiatRatesDownloader: FiatRatesFindLastTicker error ", err) + if err = fr.loadDailyTickers(); err != nil { + glog.Error("FiatRatesDownloader: loadDailyTickers error ", err) } else { - glog.Infof("FiatRatesDownloader: UpdateHistoricalTickers finished, last ticker from %v", ticker.Timestamp) - if is != nil { - is.HistoricalFiatRatesTime = ticker.Timestamp + ticker, found := fr.dailyTickers[fr.dailyTickersTo] + if !found || ticker == nil { + glog.Error("FiatRatesDownloader: dailyTickers not loaded") + } else { + glog.Infof("FiatRatesDownloader: UpdateHistoricalTickers finished, last ticker from %v", ticker.Timestamp) + fr.logTickersInfo() + if is != nil { + is.HistoricalFiatRatesTime = ticker.Timestamp + } } } if fr.downloadTokens { diff --git a/server/public_test.go b/server/public_test.go index 2d04512b..8f01c468 100644 --- a/server/public_test.go +++ b/server/public_test.go @@ -168,12 +168,12 @@ func newPostRequest(u string, body string) *http.Request { } func insertFiatRate(date string, rates map[string]float32, tokenRates map[string]float32, d *db.RocksDB) error { - convertedDate, err := db.FiatRatesConvertDate(date) + convertedDate, err := time.Parse("20060102150405", date) if err != nil { return err } ticker := &common.CurrencyRatesTicker{ - Timestamp: *convertedDate, + Timestamp: convertedDate, Rates: rates, TokenRates: tokenRates, }