Get hourly and five minutes fiat rates

db/fiat.go

@@ -84,20 +84,6 @@ func unpackCurrencyRatesTicker(buf []byte) (*common.CurrencyRatesTicker, error)
 	return &ticker, nil
 }
 
-// FiatRatesConvertDate checks if the date is in correct format and returns the Time object.
-// Possible formats are: YYYYMMDDhhmmss, YYYYMMDDhhmm, YYYYMMDDhh, YYYYMMDD
-func FiatRatesConvertDate(date string) (*time.Time, error) {
-	for format := FiatRatesTimeFormat; len(format) >= 8; format = format[:len(format)-2] {
-		convertedDate, err := time.Parse(format, date)
-		if err == nil {
-			return &convertedDate, nil
-		}
-	}
-	msg := "Date \"" + date + "\" does not match any of available formats. "
-	msg += "Possible formats are: YYYYMMDDhhmmss, YYYYMMDDhhmm, YYYYMMDDhh, YYYYMMDD"
-	return nil, errors.New(msg)
-}
-
 // FiatRatesStoreTicker stores ticker data at the specified time
 func (d *RocksDB) FiatRatesStoreTicker(wb *grocksdb.WriteBatch, ticker *common.CurrencyRatesTicker) error {
 	if len(ticker.Rates) == 0 {
@@ -145,22 +131,6 @@ func (d *RocksDB) FiatRatesGetTicker(tickerTime *time.Time) (*common.CurrencyRat
 
 // FiatRatesFindTicker gets FiatRates data closest to the specified timestamp, of the base currency, vsCurrency or the token if specified
 func (d *RocksDB) FiatRatesFindTicker(tickerTime *time.Time, vsCurrency string, token string) (*common.CurrencyRatesTicker, error) {
-	// currentTicker := d.is.GetCurrentTicker("", "")
-	// lastTickerInDBMux.Lock()
-	// dbTicker := lastTickerInDB
-	// lastTickerInDBMux.Unlock()
-	// if currentTicker != nil {
-	// 	if !tickerTime.Before(currentTicker.Timestamp) || (dbTicker != nil && tickerTime.After(dbTicker.Timestamp)) {
-	// 		f := true
-	// 		if token != "" && currentTicker.TokenRates != nil {
-	// 			_, f = currentTicker.TokenRates[token]
-	// 		}
-	// 		if f {
-	// 			return currentTicker, nil
-	// 		}
-	// 	}
-	// }
-
 	tickerTimeFormatted := tickerTime.UTC().Format(FiatRatesTimeFormat)
 	it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates])
 	defer it.Close()
@@ -178,6 +148,26 @@ func (d *RocksDB) FiatRatesFindTicker(tickerTime *time.Time, vsCurrency string,
 	return nil, nil
 }
 
+// FiatRatesGetAllTickers gets FiatRates data closest to the specified timestamp, of the base currency, vsCurrency or the token if specified
+func (d *RocksDB) FiatRatesGetAllTickers(fn func(ticker *common.CurrencyRatesTicker) error) error {
+	it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates])
+	defer it.Close()
+
+	for it.SeekToFirst(); it.Valid(); it.Next() {
+		ticker, err := getTickerFromIterator(it, "", "")
+		if err != nil {
+			return err
+		}
+		if ticker == nil {
+			return errors.New("FiatRatesGetAllTickers got nil ticker")
+		}
+		if err = fn(ticker); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // FiatRatesFindLastTicker gets the last FiatRates record, of the base currency, vsCurrency or the token if specified
 func (d *RocksDB) FiatRatesFindLastTicker(vsCurrency string, token string) (*common.CurrencyRatesTicker, error) {
 	it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates])

db/fiat_test.go

@@ -17,22 +17,6 @@ func TestRocksTickers(t *testing.T) {
 	})
 	defer closeAndDestroyRocksDB(t, d)
 
-	// Test valid formats
-	for _, date := range []string{"20190130", "2019013012", "201901301250", "20190130125030"} {
-		_, err := FiatRatesConvertDate(date)
-		if err != nil {
-			t.Errorf("%v", err)
-		}
-	}
-
-	// Test invalid formats
-	for _, date := range []string{"01102019", "10201901", "", "abc", "20190130xxx"} {
-		_, err := FiatRatesConvertDate(date)
-		if err == nil {
-			t.Errorf("Wrongly-formatted date \"%v\" marked as valid!", date)
-		}
-	}
-
 	// Test storing & finding tickers
 	pastKey, _ := time.Parse(FiatRatesTimeFormat, "20190627000000")
 	futureKey, _ := time.Parse(FiatRatesTimeFormat, "20190630000000")

fiat/coingecko.go

@@ -3,7 +3,7 @@ package fiat
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -86,7 +86,7 @@ func doReq(req *http.Request, client *http.Client) ([]byte, error) {
 		return nil, err
 	}
 	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -299,14 +299,43 @@ func (cg *Coingecko) CurrentTickers() (*common.CurrencyRatesTicker, error) {
 	return &newTickers, nil
 }
 
+func (cg *Coingecko) getHighGranularityTickers(days string) (*[]common.CurrencyRatesTicker, error) {
+	mc, err := cg.coinMarketChart(cg.coin, highGranularityVsCurrency, days, false)
+	if err != nil {
+		return nil, err
+	}
+	if len(mc.Prices) < 2 {
+		return nil, nil
+	}
+	// ignore the last point, it is not in granularity
+	tickers := make([]common.CurrencyRatesTicker, len(mc.Prices)-1)
+	for i, p := range mc.Prices[:len(mc.Prices)-1] {
+		var timestamp uint
+		timestamp = uint(p[0])
+		if timestamp > 100000000000 {
+			// convert timestamp from milliseconds to seconds
+			timestamp /= 1000
+		}
+		rate := float32(p[1])
+		u := time.Unix(int64(timestamp), 0).UTC()
+		ticker := common.CurrencyRatesTicker{
+			Timestamp: u,
+			Rates:     make(map[string]float32),
+		}
+		ticker.Rates[highGranularityVsCurrency] = rate
+		tickers[i] = ticker
+	}
+	return &tickers, nil
+}
+
 // HourlyTickers returns the array of the exchange rates in hourly granularity
 func (cg *Coingecko) HourlyTickers() (*[]common.CurrencyRatesTicker, error) {
-	return nil, nil
+	return cg.getHighGranularityTickers("90")
 }
 
 // HourlyTickers returns the array of the exchange rates in five minutes granularity
 func (cg *Coingecko) FiveMinutesTickers() (*[]common.CurrencyRatesTicker, error) {
-	return nil, nil
+	return cg.getHighGranularityTickers("1")
 }
 
 func (cg *Coingecko) getHistoricalTicker(tickersToUpdate map[uint]*common.CurrencyRatesTicker, coinId string, vsCurrency string, token string) (bool, error) {

fiat/fiat_rates.go

@@ -4,8 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -15,14 +15,20 @@ import (
 	"github.com/trezor/blockbook/db"
 )
 
-const CurrentTickersKey = "CurrentTickers"
+const currentTickersKey = "CurrentTickers"
-const HourlyTickersKey = "HourlyTickers"
+const hourlyTickersKey = "HourlyTickers"
-const FiveMinutesTickersKey = "FiveMinutesTickers"
+const fiveMinutesTickersKey = "FiveMinutesTickers"
+
+const highGranularityVsCurrency = "usd"
+
+const secondsInDay = 24 * 60 * 60
+const secondsInHour = 60 * 60
+const secondsInFiveMinutes = 5 * 60
 
 // OnNewFiatRatesTicker is used to send notification about a new FiatRates ticker
 type OnNewFiatRatesTicker func(ticker *common.CurrencyRatesTicker)
 
-// RatesDownloaderInterface provides method signatures for specific fiat rates downloaders
+// RatesDownloaderInterface provides method signatures for a specific fiat rates downloader
 type RatesDownloaderInterface interface {
 	CurrentTickers() (*common.CurrencyRatesTicker, error)
 	HourlyTickers() (*[]common.CurrencyRatesTicker, error)
@@ -55,17 +61,6 @@ type FiatRates struct {
 	dailyTickersTo         int64
 }
 
-func tickersToMap(tickers *[]common.CurrencyRatesTicker, granularitySeconds int64) (map[int64]*common.CurrencyRatesTicker, int64, int64) {
-	if tickers == nil || len(*tickers) == 0 {
-		return nil, 0, 0
-	}
-	halfGranularity := granularitySeconds / 2
-	m := make(map[int64]*common.CurrencyRatesTicker, len(*tickers))
-	from := ((*tickers)[0].Timestamp.UTC().Unix() + halfGranularity) % granularitySeconds
-	to := ((*tickers)[len(*tickers)-1].Timestamp.UTC().Unix() + halfGranularity) % granularitySeconds
-	return m, from, to
-}
-
 // NewFiatRates initializes the FiatRates handler
 func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTicker) (*FiatRates, error) {
 	var config struct {
@@ -73,7 +68,7 @@ func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTick
 		FiatRatesParams       string `json:"fiat_rates_params"`
 		FiatRatesVsCurrencies string `json:"fiat_rates_vs_currencies"`
 	}
-	data, err := ioutil.ReadFile(configFile)
+	data, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("error reading file %v, %v", configFile, err)
 	}
@@ -133,7 +128,11 @@ func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTick
 			is.HasTokenFiatRates = fr.downloadTokens
 			fr.Enabled = true
 
-			currentTickers, err := db.FiatRatesGetSpecialTickers(CurrentTickersKey)
+			if err := fr.loadDailyTickers(); err != nil {
+				return nil, err
+			}
+
+			currentTickers, err := db.FiatRatesGetSpecialTickers(currentTickersKey)
 			if err != nil {
 				glog.Error("FiatRatesDownloader: get CurrentTickers from DB error ", err)
 			}
@@ -141,22 +140,23 @@ func NewFiatRates(db *db.RocksDB, configFile string, callback OnNewFiatRatesTick
 				fr.currentTicker = &(*currentTickers)[0]
 			}
 
-			hourlyTickers, err := db.FiatRatesGetSpecialTickers(HourlyTickersKey)
+			hourlyTickers, err := db.FiatRatesGetSpecialTickers(hourlyTickersKey)
 			if err != nil {
 				glog.Error("FiatRatesDownloader: get HourlyTickers from DB error ", err)
 			}
-			fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = tickersToMap(hourlyTickers, 3600)
+			fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = fr.tickersToMap(hourlyTickers, secondsInHour)
 
-			fiveMinutesTickers, err := db.FiatRatesGetSpecialTickers(FiveMinutesTickersKey)
+			fiveMinutesTickers, err := db.FiatRatesGetSpecialTickers(fiveMinutesTickersKey)
 			if err != nil {
 				glog.Error("FiatRatesDownloader: get FiveMinutesTickers from DB error ", err)
 			}
-			fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = tickersToMap(fiveMinutesTickers, 5*60)
+			fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = fr.tickersToMap(fiveMinutesTickers, secondsInFiveMinutes)
 
 		}
 	} else {
 		return nil, fmt.Errorf("unknown provider %q", fr.provider)
 	}
+	fr.logTickersInfo()
 	return fr, nil
 }
 
@@ -171,28 +171,208 @@ func (fr *FiatRates) GetCurrentTicker(vsCurrency string, token string) *common.C
 	return nil
 }
 
+// getTokenTickersForTimestamps returns tickers for slice of timestamps, that contain requested vsCurrency and token
+func (fr *FiatRates) getTokenTickersForTimestamps(timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) {
+	currentTicker := fr.GetCurrentTicker("", token)
+	tickers := make([]*common.CurrencyRatesTicker, len(timestamps))
+	var prevTicker *common.CurrencyRatesTicker
+	for i, t := range timestamps {
+		// check if the token is available in the current ticker - if not, return nil ticker instead of wasting time in costly DB searches
+		if currentTicker != nil {
+			var ticker *common.CurrencyRatesTicker
+			date := time.Unix(t, 0)
+			// if previously found ticker is newer than this one (token tickers may not be in DB for every day), skip search in DB
+			if prevTicker != nil && !date.After(prevTicker.Timestamp) {
+				ticker = prevTicker
+			} else {
+				ticker, _ := fr.db.FiatRatesFindTicker(&date, vsCurrency, token)
+				prevTicker = ticker
+			}
+			// if ticker not found in DB, use current ticker
+			if ticker == nil {
+				tickers[i] = currentTicker
+				prevTicker = currentTicker
+			} else {
+				tickers[i] = ticker
+			}
+		}
+	}
+	return &tickers, nil
+}
+
+// GetTickersForTimestamps returns tickers for slice of timestamps, that contain requested vsCurrency and token
+func (fr *FiatRates) GetTickersForTimestamps(timestamps []int64, vsCurrency string, token string) (*[]*common.CurrencyRatesTicker, error) {
+	// token rates are not in memory, them load from DB
+	if token != "" {
+		return fr.getTokenTickersForTimestamps(timestamps, vsCurrency, token)
+	}
+	fr.mux.RLock()
+	defer fr.mux.RUnlock()
+	tickers := make([]*common.CurrencyRatesTicker, len(timestamps))
+	var prevTicker *common.CurrencyRatesTicker
+	for i, t := range timestamps {
+		dailyTs := normalizedUnix(t, secondsInDay)
+		// use higher granularity only for non daily timestamps
+		if t != dailyTs {
+			if t >= fr.fiveMinutesTickersFrom && t <= fr.fiveMinutesTickersTo {
+				if ticker, found := fr.fiveMinutesTickers[normalizedUnix(t, secondsInFiveMinutes)]; found && ticker != nil {
+					if common.IsSuitableTicker(ticker, vsCurrency, token) {
+						tickers[i] = ticker
+						continue
+					}
+				}
+			}
+			if t >= fr.hourlyTickersFrom && t <= fr.hourlyTickersTo {
+				if ticker, found := fr.hourlyTickers[normalizedUnix(t, secondsInHour)]; found && ticker != nil {
+					if common.IsSuitableTicker(ticker, vsCurrency, token) {
+						tickers[i] = ticker
+						continue
+					}
+				}
+			}
+		}
+		if prevTicker != nil && dailyTs >= prevTicker.Timestamp.Unix() {
+			tickers[i] = prevTicker
+			continue
+		} else {
+			var found bool
+			if dailyTs < fr.dailyTickersFrom {
+				dailyTs = fr.dailyTickersFrom
+			}
+			var ticker *common.CurrencyRatesTicker
+			for ; dailyTs <= fr.dailyTickersTo; dailyTs += secondsInDay {
+				if ticker, found = fr.dailyTickers[dailyTs]; found && ticker != nil {
+					if common.IsSuitableTicker(ticker, vsCurrency, token) {
+						tickers[i] = ticker
+						prevTicker = ticker
+						break
+					} else {
+						found = false
+					}
+				}
+				if !found {
+					tickers[i] = fr.currentTicker
+					prevTicker = fr.currentTicker
+				}
+			}
+		}
+	}
+	return &tickers, nil
+}
+func (fr *FiatRates) logTickersInfo() {
+	glog.Infof("fiat rates %s handler, %d (%s - %s) daily tickers, %d (%s - %s) hourly tickers, %d (%s - %s) 5 minute tickers", fr.provider,
+		len(fr.dailyTickers), time.Unix(fr.dailyTickersFrom, 0).Format("2006-01-02"), time.Unix(fr.dailyTickersTo, 0).Format("2006-01-02"),
+		len(fr.hourlyTickers), time.Unix(fr.hourlyTickersFrom, 0).Format("2006-01-02 15:04"), time.Unix(fr.hourlyTickersTo, 0).Format("2006-01-02 15:04"),
+		len(fr.fiveMinutesTickers), time.Unix(fr.fiveMinutesTickersFrom, 0).Format("2006-01-02 15:04"), time.Unix(fr.fiveMinutesTickersTo, 0).Format("2006-01-02 15:04"))
+}
+
+func normalizedTimeUnix(t time.Time, granularity int64) int64 {
+	return normalizedUnix(t.UTC().Unix(), granularity)
+}
+
+func normalizedUnix(t int64, granularity int64) int64 {
+	unix := t + (granularity >> 1)
+	return unix - unix%granularity
+}
+
+// loadDailyTickers loads daily tickers to cache
+func (fr *FiatRates) loadDailyTickers() error {
+	fr.mux.Lock()
+	defer fr.mux.Unlock()
+	fr.dailyTickers = make(map[int64]*common.CurrencyRatesTicker)
+	err := fr.db.FiatRatesGetAllTickers(func(ticker *common.CurrencyRatesTicker) error {
+		normalizedTime := normalizedTimeUnix(ticker.Timestamp, secondsInDay)
+		// remove token rates from cache to save memory (tickers with token rates are hundreds of kb big)
+		ticker.TokenRates = nil
+		if len(fr.dailyTickers) > 0 {
+			// check that there is a ticker for every day, if missing, set it from current value if missing
+			prevTime := normalizedTime
+			for {
+				prevTime -= secondsInDay
+				if _, found := fr.dailyTickers[prevTime]; found {
+					break
+				}
+				fr.dailyTickers[prevTime] = ticker
+			}
+		} else {
+			fr.dailyTickersFrom = normalizedTime
+		}
+		fr.dailyTickers[normalizedTime] = ticker
+		fr.dailyTickersTo = normalizedTime
+		return nil
+	})
+	return err
+}
+
 // setCurrentTicker sets current ticker
 func (fr *FiatRates) setCurrentTicker(t *common.CurrencyRatesTicker) {
 	fr.mux.Lock()
 	defer fr.mux.Unlock()
 	fr.currentTicker = t
-	fr.db.FiatRatesStoreSpecialTickers(CurrentTickersKey, &[]common.CurrencyRatesTicker{*t})
+	fr.db.FiatRatesStoreSpecialTickers(currentTickersKey, &[]common.CurrencyRatesTicker{*t})
+}
+
+func (fr *FiatRates) tickersToMap(tickers *[]common.CurrencyRatesTicker, granularitySeconds int64) (map[int64]*common.CurrencyRatesTicker, int64, int64) {
+	if tickers == nil || len(*tickers) == 0 {
+		return make(map[int64]*common.CurrencyRatesTicker), 0, 0
+	}
+	m := make(map[int64]*common.CurrencyRatesTicker, len(*tickers))
+	from := int64(0)
+	to := int64(0)
+	for i := range *tickers {
+		ticker := (*tickers)[i]
+		normalizedTime := normalizedTimeUnix(ticker.Timestamp, granularitySeconds)
+		dailyTime := normalizedTimeUnix(ticker.Timestamp, secondsInDay)
+		dailyTicker, found := fr.dailyTickers[dailyTime]
+		if !found {
+			// if not found in historical tickers, use current ticker
+			dailyTicker = fr.currentTicker
+		}
+		if dailyTicker != nil {
+			// high granularity tickers are loaded only in one currency, add other currencies based on daily rate between fiat currencies
+			vsRate, foundVs := ticker.Rates[highGranularityVsCurrency]
+			dailyVsRate, foundDaily := dailyTicker.Rates[highGranularityVsCurrency]
+			if foundDaily && dailyVsRate != 0 && foundVs && vsRate != 0 {
+				for currency, rate := range dailyTicker.Rates {
+					if currency != highGranularityVsCurrency {
+						ticker.Rates[currency] = vsRate * rate / dailyVsRate
+					}
+				}
+			}
+		}
+		if len(m) > 0 {
+			// check that there is a ticker for each period, set it from current value if missing
+			prevTime := normalizedTime
+			for {
+				prevTime -= granularitySeconds
+				if _, found := m[prevTime]; found {
+					break
+				}
+				m[prevTime] = &ticker
+			}
+		} else {
+			from = normalizedTime
+		}
+		m[normalizedTime] = &ticker
+		to = normalizedTime
+	}
+	return m, from, to
 }
 
 // setCurrentTicker sets hourly tickers
 func (fr *FiatRates) setHourlyTickers(t *[]common.CurrencyRatesTicker) {
+	fr.db.FiatRatesStoreSpecialTickers(hourlyTickersKey, t)
 	fr.mux.Lock()
 	defer fr.mux.Unlock()
-	fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = tickersToMap(t, 3600)
+	fr.hourlyTickers, fr.hourlyTickersFrom, fr.hourlyTickersTo = fr.tickersToMap(t, secondsInHour)
-	fr.db.FiatRatesStoreSpecialTickers(HourlyTickersKey, t)
 }
 
 // setCurrentTicker sets hourly tickers
 func (fr *FiatRates) setFiveMinutesTickers(t *[]common.CurrencyRatesTicker) {
+	fr.db.FiatRatesStoreSpecialTickers(fiveMinutesTickersKey, t)
 	fr.mux.Lock()
 	defer fr.mux.Unlock()
-	fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = tickersToMap(t, 5*60)
+	fr.fiveMinutesTickers, fr.fiveMinutesTickersFrom, fr.fiveMinutesTickersTo = fr.tickersToMap(t, secondsInFiveMinutes)
-	fr.db.FiatRatesStoreSpecialTickers(FiveMinutesTickersKey, t)
 }
 
 // RunDownloader periodically downloads current (every 15 minutes) and historical (once a day) tickers
@@ -245,13 +425,18 @@ func (fr *FiatRates) RunDownloader() error {
 				glog.Error("FiatRatesDownloader: UpdateHistoricalTickers error ", err)
 			} else {
 				lastHistoricalTickers = time.Now().UTC()
-				ticker, err := fr.db.FiatRatesFindLastTicker("", "")
+				if err = fr.loadDailyTickers(); err != nil {
-				if err != nil || ticker == nil {
+					glog.Error("FiatRatesDownloader: loadDailyTickers error ", err)
-					glog.Error("FiatRatesDownloader: FiatRatesFindLastTicker error ", err)
 				} else {
-					glog.Infof("FiatRatesDownloader: UpdateHistoricalTickers finished, last ticker from %v", ticker.Timestamp)
+					ticker, found := fr.dailyTickers[fr.dailyTickersTo]
-					if is != nil {
+					if !found || ticker == nil {
-						is.HistoricalFiatRatesTime = ticker.Timestamp
+						glog.Error("FiatRatesDownloader: dailyTickers not loaded")
+					} else {
+						glog.Infof("FiatRatesDownloader: UpdateHistoricalTickers finished, last ticker from %v", ticker.Timestamp)
+						fr.logTickersInfo()
+						if is != nil {
+							is.HistoricalFiatRatesTime = ticker.Timestamp
+						}
 					}
 				}
 				if fr.downloadTokens {

server/public_test.go

@@ -168,12 +168,12 @@ func newPostRequest(u string, body string) *http.Request {
 }
 
 func insertFiatRate(date string, rates map[string]float32, tokenRates map[string]float32, d *db.RocksDB) error {
-	convertedDate, err := db.FiatRatesConvertDate(date)
+	convertedDate, err := time.Parse("20060102150405", date)
 	if err != nil {
 		return err
 	}
 	ticker := &common.CurrencyRatesTicker{
-		Timestamp:  *convertedDate,
+		Timestamp:  convertedDate,
 		Rates:      rates,
 		TokenRates: tokenRates,
 	}