Add prometheus metrics

blockbook.go

@@ -469,6 +469,8 @@ func blockbookAppInfoMetric(db *db.RocksDB, chain bchain.BlockChain, txCache *db
 		"backend_version":          si.Backend.Version,
 		"backend_subversion":       si.Backend.Subversion,
 		"backend_protocol_version": si.Backend.ProtocolVersion}).Set(float64(0))
+	metrics.BackendBestHeight.Set(float64(si.Backend.Blocks))
+	metrics.BlockbookBestHeight.Set(float64(si.Blockbook.BestHeight))
 	return nil
 }
 

common/metrics.go

@@ -8,25 +8,30 @@ import (
 
 // Metrics holds prometheus collectors for various metrics collected by Blockbook
 type Metrics struct {
-	SocketIORequests      *prometheus.CounterVec
+	SocketIORequests         *prometheus.CounterVec
-	SocketIOSubscribes    *prometheus.CounterVec
+	SocketIOSubscribes       *prometheus.CounterVec
-	SocketIOClients       prometheus.Gauge
+	SocketIOClients          prometheus.Gauge
-	SocketIOReqDuration   *prometheus.HistogramVec
+	SocketIOReqDuration      *prometheus.HistogramVec
-	WebsocketRequests     *prometheus.CounterVec
+	WebsocketRequests        *prometheus.CounterVec
-	WebsocketSubscribes   *prometheus.CounterVec
+	WebsocketSubscribes      *prometheus.CounterVec
-	WebsocketClients      prometheus.Gauge
+	WebsocketClients         prometheus.Gauge
-	WebsocketReqDuration  *prometheus.HistogramVec
+	WebsocketReqDuration     *prometheus.HistogramVec
-	IndexResyncDuration   prometheus.Histogram
+	IndexResyncDuration      prometheus.Histogram
-	MempoolResyncDuration prometheus.Histogram
+	MempoolResyncDuration    prometheus.Histogram
-	TxCacheEfficiency     *prometheus.CounterVec
+	TxCacheEfficiency        *prometheus.CounterVec
-	RPCLatency            *prometheus.HistogramVec
+	RPCLatency               *prometheus.HistogramVec
-	IndexResyncErrors     *prometheus.CounterVec
+	IndexResyncErrors        *prometheus.CounterVec
-	IndexDBSize           prometheus.Gauge
+	IndexDBSize              prometheus.Gauge
-	ExplorerViews         *prometheus.CounterVec
+	ExplorerViews            *prometheus.CounterVec
-	MempoolSize           prometheus.Gauge
+	MempoolSize              prometheus.Gauge
-	DbColumnRows          *prometheus.GaugeVec
+	DbColumnRows             *prometheus.GaugeVec
-	DbColumnSize          *prometheus.GaugeVec
+	DbColumnSize             *prometheus.GaugeVec
-	BlockbookAppInfo      *prometheus.GaugeVec
+	BlockbookAppInfo         *prometheus.GaugeVec
+	BackendBestHeight        prometheus.Gauge
+	BlockbookBestHeight      prometheus.Gauge
+	ExplorerPendingRequests  *prometheus.GaugeVec
+	WebsocketPendingRequests *prometheus.GaugeVec
+	SocketIOPendingRequests  *prometheus.GaugeVec
 }
 
 // Labels represents a collection of label name -> value mappings.
@@ -187,6 +192,44 @@ func GetMetrics(coin string) (*Metrics, error) {
 		},
 		[]string{"blockbook_version", "blockbook_commit", "blockbook_buildtime", "backend_version", "backend_subversion", "backend_protocol_version"},
 	)
+	metrics.BlockbookBestHeight = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name:        "blockbook_best_height",
+			Help:        "Block height in Blockbook",
+			ConstLabels: Labels{"coin": coin},
+		},
+	)
+	metrics.BackendBestHeight = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name:        "blockbook_backend_best_height",
+			Help:        "Block height in backend",
+			ConstLabels: Labels{"coin": coin},
+		},
+	)
+	metrics.ExplorerPendingRequests = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "blockbook_explorer_pending_reqests",
+			Help:        "Number of unfinished requests in explorer interface",
+			ConstLabels: Labels{"coin": coin},
+		},
+		[]string{"method"},
+	)
+	metrics.WebsocketPendingRequests = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "blockbook_websocket_pending_reqests",
+			Help:        "Number of unfinished requests in websocket interface",
+			ConstLabels: Labels{"coin": coin},
+		},
+		[]string{"method"},
+	)
+	metrics.SocketIOPendingRequests = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "blockbook_socketio_pending_reqests",
+			Help:        "Number of unfinished requests in socketio interface",
+			ConstLabels: Labels{"coin": coin},
+		},
+		[]string{"method"},
+	)
 
 	v := reflect.ValueOf(metrics)
 	for i := 0; i < v.NumField(); i++ {

db/sync.go

@@ -95,6 +95,8 @@ func (w *SyncWorker) ResyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync b
 		if err == nil {
 			w.is.FinishedSync(bh)
 		}
+		w.metrics.BackendBestHeight.Set(float64(w.is.BackendInfo.Blocks))
+		w.metrics.BlockbookBestHeight.Set(float64(bh))
 		return err
 	case errSynced:
 		// this is not actually error but flag that resync wasn't necessary
@@ -226,6 +228,7 @@ func (w *SyncWorker) connectBlocks(onNewBlock bchain.OnNewBlockFunc, initialSync
 		if onNewBlock != nil {
 			onNewBlock(res.block.Hash, res.block.Height)
 		}
+		w.metrics.BlockbookBestHeight.Set(float64(res.block.Height))
 		if res.block.Height > 0 && res.block.Height%1000 == 0 {
 			glog.Info("connected block ", res.block.Height, " ", res.block.Hash)
 		}
@@ -377,6 +380,7 @@ ConnectLoop:
 			}
 			hch <- hashHeight{hash, h}
 			if h > 0 && h%1000 == 0 {
+				w.metrics.BlockbookBestHeight.Set(float64(h))
 				glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start), " ", w.db.GetAndResetConnectBlockStats())
 				start = time.Now()
 			}

server/public.go

@@ -258,7 +258,13 @@ func joinURL(base string, part string) string {
 }
 
 func getFunctionName(i interface{}) string {
-	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
+	name := runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
+	start := strings.LastIndex(name, ".")
+	end := strings.LastIndex(name, "-")
+	if start > 0 && end > start {
+		name = name[start+1 : end]
+	}
+	return name
 }
 
 func (s *PublicServer) jsonHandler(handler func(r *http.Request, apiVersion int) (interface{}, error), apiVersion int) func(w http.ResponseWriter, r *http.Request) {
@@ -266,12 +272,13 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request, apiVersion int)
 		Text       string `json:"error"`
 		HTTPStatus int    `json:"-"`
 	}
+	handlerName := getFunctionName(handler)
 	return func(w http.ResponseWriter, r *http.Request) {
 		var data interface{}
 		var err error
 		defer func() {
 			if e := recover(); e != nil {
-				glog.Error(getFunctionName(handler), " recovered from panic: ", e)
+				glog.Error(handlerName, " recovered from panic: ", e)
 				debug.PrintStack()
 				if s.debug {
 					data = jsonError{fmt.Sprint("Internal server error: recovered from panic ", e), http.StatusInternalServerError}
@@ -287,7 +294,9 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request, apiVersion int)
 			if err != nil {
 				glog.Warning("json encode ", err)
 			}
+			s.metrics.ExplorerPendingRequests.With((common.Labels{"method": handlerName})).Dec()
 		}()
+		s.metrics.ExplorerPendingRequests.With((common.Labels{"method": handlerName})).Inc()
 		data, err = handler(r, apiVersion)
 		if err != nil || data == nil {
 			if apiErr, ok := err.(*api.APIError); ok {
@@ -298,7 +307,7 @@ func (s *PublicServer) jsonHandler(handler func(r *http.Request, apiVersion int)
 				}
 			} else {
 				if err != nil {
-					glog.Error(getFunctionName(handler), " error: ", err)
+					glog.Error(handlerName, " error: ", err)
 				}
 				if s.debug {
 					if data != nil {
@@ -332,13 +341,14 @@ func (s *PublicServer) newTemplateDataWithError(text string) *TemplateData {
 }
 
 func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r *http.Request) (tpl, *TemplateData, error)) func(w http.ResponseWriter, r *http.Request) {
+	handlerName := getFunctionName(handler)
 	return func(w http.ResponseWriter, r *http.Request) {
 		var t tpl
 		var data *TemplateData
 		var err error
 		defer func() {
 			if e := recover(); e != nil {
-				glog.Error(getFunctionName(handler), " recovered from panic: ", e)
+				glog.Error(handlerName, " recovered from panic: ", e)
 				debug.PrintStack()
 				t = errorInternalTpl
 				if s.debug {
@@ -358,7 +368,9 @@ func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r
 					glog.Error(err)
 				}
 			}
+			s.metrics.ExplorerPendingRequests.With((common.Labels{"method": handlerName})).Dec()
 		}()
+		s.metrics.ExplorerPendingRequests.With((common.Labels{"method": handlerName})).Inc()
 		if s.debug {
 			// reload templates on each request
 			// to reflect changes during development
@@ -375,7 +387,7 @@ func (s *PublicServer) htmlTemplateHandler(handler func(w http.ResponseWriter, r
 				}
 			} else {
 				if err != nil {
-					glog.Error(getFunctionName(handler), " error: ", err)
+					glog.Error(handlerName, " error: ", err)
 				}
 				if s.debug {
 					data = s.newTemplateDataWithError(fmt.Sprintf("Internal server error: %v, data %+v", err, data))

server/socketio.go

@@ -169,9 +169,11 @@ func (s *SocketIoServer) onMessage(c *gosocketio.Channel, req map[string]json.Ra
 			e.Error.Message = "Internal error"
 			rv = e
 		}
+		s.metrics.SocketIOPendingRequests.With((common.Labels{"method": method})).Dec()
 	}()
 	t := time.Now()
 	params := req["params"]
+	s.metrics.SocketIOPendingRequests.With((common.Labels{"method": method})).Inc()
 	defer s.metrics.SocketIOReqDuration.With(common.Labels{"method": method}).Observe(float64(time.Since(t)) / 1e3) // in microseconds
 	f, ok := onMessageHandlers[method]
 	if ok {

server/websocket.go

@@ -436,8 +436,10 @@ func (s *WebsocketServer) onRequest(c *websocketChannel, req *websocketReq) {
 				Data: data,
 			})
 		}
+		s.metrics.WebsocketPendingRequests.With((common.Labels{"method": req.Method})).Dec()
 	}()
 	t := time.Now()
+	s.metrics.WebsocketPendingRequests.With((common.Labels{"method": req.Method})).Inc()
 	defer s.metrics.WebsocketReqDuration.With(common.Labels{"method": req.Method}).Observe(float64(time.Since(t)) / 1e3) // in microseconds
 	f, ok := requestHandlers[req.Method]
 	if ok {