feat(metric): upgrade metrics

Author: amazing-gao

docs/grafana_dashboard.json

@@ -18,7 +18,7 @@ var (
 	boxMetricGauge = metric.Default.NewGaugeVec(
 		"box_info",
 		"Information about the box config and environment.",
-		[]string{"name", "version", "tags", "ip", "localhost", "start"})
+		[]string{"tags", "ip", "localhost", "start"})
 )
 
 func (boxMetric) Name() string {
@@ -27,8 +27,6 @@ func (boxMetric) Name() string {
 
 func (boxMetric) Serve(ctx context.Context) error {
 	boxMetricGauge.WithLabelValues(
-		config.ServiceName(),
-		config.ServiceVersion(),
 		strings.Join(config.ServiceTag(), ","),
 		system.IP(),
 		system.Hostname(),

docs/metric.md

@@ -24,19 +24,19 @@ const (
 )
 
 var (
-	metricConnIdle     = metric.NewGaugeVec("db_connections_idle", `The number of idle connections.`, []string{labelDriver, labelDatabase})
-	metricConnInUse    = metric.NewGaugeVec("db_connections_in_use", `The number of connections currently in use.`, []string{labelDriver, labelDatabase})
-	metricConnOpen     = metric.NewGaugeVec("db_connections_open", `The number of established connections both in use and idle.`, []string{labelDriver, labelDatabase})
-	metricConnMaxOpen  = metric.NewGaugeVec("db_connections_max_open", `Maximum number of open connections to the database.`, []string{labelDriver, labelDatabase})
-	metricWaitCount    = metric.NewGaugeVec("db_wait_count", `The total number of connections waited for.`, []string{labelDriver, labelDatabase})
-	metricWaitDuration = metric.NewGaugeVec("db_wait_duration_seconds", `The total time blocked waiting for a new connection.`, []string{labelDriver, labelDatabase})
-	metricSQLSeconds   = metric.NewSummaryVec("db_sql_seconds", `All queries requested seconds`, []string{labelDriver, labelDatabase, labelType, labelError}, map[float64]float64{
-		0.5:  0.05,
-		0.75: 0.05,
-		0.9:  0.01,
-		0.99: 0.001,
-		1:    0.001,
-	})
+	metricConnIdle     = metric.NewGaugeVec("db_client_connections_idle", `The number of idle connections.`, []string{labelDriver, labelDatabase})
+	metricConnInUse    = metric.NewGaugeVec("db_client_connections_in_use", `The number of connections currently in use.`, []string{labelDriver, labelDatabase})
+	metricConnOpen     = metric.NewGaugeVec("db_client_connections_open", `The number of established connections both in use and idle.`, []string{labelDriver, labelDatabase})
+	metricConnMaxOpen  = metric.NewGaugeVec("db_client_connections_max_open", `Maximum number of open connections to the database.`, []string{labelDriver, labelDatabase})
+	metricWaitCount    = metric.NewGaugeVec("db_client_connections_wait_total", `The total number of connections waited for.`, []string{labelDriver, labelDatabase})
+	metricWaitDuration = metric.NewGaugeVec("db_client_connections_wait_seconds", `The total time blocked waiting for a new connection.`, []string{labelDriver, labelDatabase})
+	metricSQLDuration  = metric.NewHistogramVec(
+		"db_client_request_duration_seconds",
+		"The SQL execution latencies in seconds.",
+		[]string{labelDriver, labelDatabase, labelType, "result"},
+		// 250us, 500us, 1ms, 2.5ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s
+		[]float64{0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5},
+	)
 )
 
 func newMetric(driver, database string, statsInterval time.Duration) *Metric {
@@ -115,11 +115,11 @@ func (m *Metric) beforeCallback(db *DB) {
 
 func (m *Metric) afterCallback(cmdType string) func(*DB) {
 	return func(db *DB) {
-		err := ""
+		result := "success"
 		second := 0.0
 
 		if db.Statement.Error != nil {
-			err = db.Statement.Error.Error()
+			result = "error"
 		}
 
 		if ts, ok := db.InstanceGet("startTime"); ok {
@@ -128,7 +128,7 @@ func (m *Metric) afterCallback(cmdType string) func(*DB) {
 			}
 		}
 
-		metricSQLSeconds.WithLabelValues(m.driver, m.database, cmdType, err).Observe(second)
+		metricSQLDuration.WithLabelValues(m.driver, m.database, cmdType, result).Observe(second)
 	}
 }
 

metric.go

@@ -19,25 +19,20 @@ type (
 
 var (
 	cmdTotal = metric.NewCounterVec(
-		"mongo_client_command_total",
-		"mongodb client command counter",
-		[]string{"command", "error"},
+		"mongo_client_requests_total",
+		"The total number of MongoDB commands executed.",
+		[]string{"command", "result"},
 	)
-	cmdDuration = metric.NewSummaryVec(
-		"mongo_client_command_duration_seconds",
-		"mongodb client command duration seconds",
-		[]string{"command", "error"},
-		map[float64]float64{
-			0.5:  0.05,
-			0.75: 0.05,
-			0.9:  0.01,
-			0.99: 0.001,
-			1:    0.001,
-		},
+	cmdDuration = metric.NewHistogramVec(
+		"mongo_client_request_duration_seconds",
+		"The MongoDB command latencies in seconds.",
+		[]string{"command", "result"},
+		// 250us, 500us, 1ms, 2.5ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s
+		[]float64{0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5},
 	)
 	workingSession = metric.NewGaugeVec(
-		"mongo_client_session_in_progress",
-		"mongo client session in progress gauge",
+		"mongo_client_sessions_inflight",
+		"The number of MongoDB sessions currently in progress.",
 		[]string{},
 	)
 )
@@ -80,15 +75,15 @@ func (mon *metricMonitor) Started(ctx context.Context, ev *event.CommandStartedE
 }
 
 func (mon *metricMonitor) Succeeded(ctx context.Context, ev *event.CommandSucceededEvent) {
-	labels := []string{ev.CommandName, ""}
+	labels := []string{ev.CommandName, "success"}
 	cmdTotal.WithLabelValues(labels...).Inc()
 	cmdDuration.WithLabelValues(labels...).Observe(time.Duration(ev.DurationNanos).Seconds())
 
 	logger.Trace(ctx).Debugf("mongo_command_success cmd: %s, reqId: %d, connId: %s, duration: %s", ev.CommandName, ev.RequestID, ev.ConnectionID, time.Duration(ev.DurationNanos))
 }
 
 func (mon *metricMonitor) Failed(ctx context.Context, ev *event.CommandFailedEvent) {
-	labels := []string{ev.CommandName, ev.Failure}
+	labels := []string{ev.CommandName, "error"}
 	cmdTotal.WithLabelValues(labels...).Inc()
 	cmdDuration.WithLabelValues(labels...).Observe(time.Duration(ev.DurationNanos).Seconds())
 

pkg/client/gormx/metric.go

@@ -10,10 +10,18 @@ import (
 
 type (
 	Logger struct {
-		cfg *Config
+		cfg  *Config
+		addr string
 	}
 )
 
+func newLogger(cfg *Config) *Logger {
+	return &Logger{
+		cfg:  cfg,
+		addr: strings.Join(cfg.Address, ","),
+	}
+}
+
 func (inst *Logger) DialHook(next redis.DialHook) redis.DialHook {
 	return next
 }
@@ -58,7 +66,7 @@ func (inst *Logger) log(ctx context.Context, pipe bool, cmds ...redis.Cmder) {
 
 	if len(errArr) > 0 {
 		logger.Trace(ctx).Errorw("Redis.Error",
-			"address", strings.Join(inst.cfg.Address, ","),
+			"address", inst.addr,
 			"db", inst.cfg.DB,
 			"err", strings.Join(errArr, ";"),
 			"cmd", strings.Join(cmdArr, ";"),

pkg/client/mongodb/metric.go

@@ -2,40 +2,42 @@ package redis
 
 import (
 	"context"
-	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/boxgo/box/pkg/metric"
-	"github.com/boxgo/box/pkg/trace"
 	"github.com/redis/go-redis/v9"
 )
 
 type (
 	Metric struct {
-		cfg *Config
+		cfg  *Config
+		addr string
 	}
 
 	startKey struct{}
 )
 
+func newMetric(cfg *Config) *Metric {
+	return &Metric{
+		cfg:  cfg,
+		addr: strings.Join(cfg.Address, ","),
+	}
+}
+
 var (
 	cmdTotal = metric.NewCounterVec(
-		"redis_client_command_total",
-		"redis command counter",
-		[]string{"bid", "address", "db", "masterName", "pipe", "cmd", "error"},
+		"redis_client_requests_total",
+		"The total number of Redis commands executed.",
+		[]string{"address", "db", "masterName", "pipe", "cmd", "result"},
 	)
-	cmdDuration = metric.NewSummaryVec(
-		"redis_client_command_duration_seconds",
-		"redis command duration seconds",
-		[]string{"bid", "address", "db", "masterName", "pipe", "cmd", "error"},
-		map[float64]float64{
-			0.5:  0.05,
-			0.75: 0.05,
-			0.9:  0.01,
-			0.99: 0.001,
-			1:    0.001,
-		},
+	cmdDuration = metric.NewHistogramVec(
+		"redis_client_request_duration_seconds",
+		"The Redis command latencies in seconds.",
+		[]string{"address", "db", "masterName", "pipe", "cmd", "result"},
+		// 100us, 250us, 500us, 1ms, 2.5ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms
+		[]float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5},
 	)
 )
 
@@ -68,38 +70,33 @@ func (m *Metric) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.Proce
 }
 
 func (m *Metric) report(ctx context.Context, pipe bool, elapsed time.Duration, cmds ...redis.Cmder) {
-	addressStr := strings.Join(m.cfg.Address, ",")
-	dbStr := fmt.Sprintf("%d", m.cfg.DB)
-	masterNameStr := m.cfg.MasterName
-	errStr := ""
 	cmdStr := ""
-	pipeStr := fmt.Sprintf("%t", pipe)
+	result := "success"
+	masterNameStr := m.cfg.MasterName
+	addressStr := m.addr
+	dbStr := strconv.Itoa(m.cfg.DB)
+	pipeStr := strconv.FormatBool(pipe)
+
+	if pipe {
+		cmdStr = "pipeline"
+	} else if len(cmds) > 0 {
+		cmdStr = cmds[0].Name()
+	}
 
 	for _, cmd := range cmds {
-		cmdStr += cmd.Name() + ";"
-
 		if err := cmd.Err(); err != nil && err != redis.Nil {
-			errStr += err.Error() + ";"
+			result = "error"
+			break
 		}
 	}
-	cmdStr = strings.TrimSuffix(cmdStr, ";")
-
-	var (
-		bizID string
-	)
-
-	if bizIDStr, ok := ctx.Value(trace.BizID()).(string); ok {
-		bizID = bizIDStr
-	}
 
 	values := []string{
-		bizID,
 		addressStr,
 		dbStr,
 		masterNameStr,
 		pipeStr,
 		cmdStr,
-		errStr,
+		result,
 	}
 
 	cmdDuration.WithLabelValues(values...).Observe(elapsed.Seconds())

pkg/client/redis/logger.go

@@ -26,8 +26,8 @@ func newRedis(cfg *Config) *Redis {
 		MinIdleConns: cfg.MinIdleConnCnt,
 	})
 
-	client.AddHook(&Metric{cfg: cfg})
-	client.AddHook(&Logger{cfg: cfg})
+	client.AddHook(newMetric(cfg))
+	client.AddHook(newLogger(cfg))
 
 	if err := redisotel.InstrumentTracing(client); err != nil {
 		logger.Panicf("Redis.InstrumentTracing.Error: %s", err)

pkg/client/redis/metric.go

@@ -1,11 +1,12 @@
 package wukong
 
 import (
+	"context"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/boxgo/box/pkg/metric"
-	"golang.org/x/net/context"
 )
 
 type (
@@ -18,36 +19,39 @@ const (
 
 var (
 	requestInflight = metric.NewGaugeVec(
-		"http_client_request_in_process",
-		"http client requesting",
+		"http_client_requests_inflight",
+		"The number of HTTP client requests currently in flight.",
 		[]string{"method", "baseUrl", "url"},
 	)
 	requestCounter = metric.NewCounterVec(
-		"http_client_request_total",
-		"http client request counter",
-		[]string{"method", "baseUrl", "url", "statusCode", "error"},
+		"http_client_requests_total",
+		"The total number of HTTP client requests sent.",
+		[]string{"method", "baseUrl", "url", "status", "error"},
 	)
-	requestDuration = metric.NewSummaryVec(
-		"http_client_request_seconds",
-		"http client request duration",
-		[]string{"method", "baseUrl", "url", "statusCode", "error"},
-		map[float64]float64{
-			0.5:  0.05,
-			0.75: 0.05,
-			0.9:  0.01,
-			0.99: 0.001,
-			1:    0.001,
-		},
+	requestDuration = metric.NewHistogramVec(
+		"http_client_request_duration_seconds",
+		"The HTTP client request latencies in seconds.",
+		[]string{"method", "baseUrl", "url", "status", "error"},
+		// 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s
+		[]float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
 	)
 )
 
+// stripQuery removes query parameters and fragment from URL
+func stripQuery(url string) string {
+	if idx := strings.IndexAny(url, "?#"); idx != -1 {
+		return url[:idx]
+	}
+	return url
+}
+
 func metricStart(request *Request) error {
 	if val, ok := request.Context.Value(metricSwitchKey).(bool); ok && !val {
 		return nil
 	}
 
-	requestInflight.WithLabelValues(request.Method, request.BaseUrl, request.Url).Inc()
-
+	url := stripQuery(request.Url)
+	requestInflight.WithLabelValues(request.Method, request.BaseUrl, url).Inc()
 	request.Context = context.WithValue(request.Context, metricDurationKey{}, time.Now())
 
 	return nil
@@ -59,22 +63,23 @@ func metricEnd(request *Request, resp *Response) error {
 	}
 
 	var (
-		errMsg     = ""
-		duration   = time.Duration(0)
-		statusCode = strconv.Itoa(resp.StatusCode())
+		errMsg   = ""
+		duration = time.Duration(0)
+		status   = strconv.Itoa(resp.StatusCode())
 	)
 
 	if resp.Error() != nil {
-		errMsg = resp.Error().Error()
+		errMsg = "error"
 	}
 
 	if start, ok := request.Context.Value(metricDurationKey{}).(time.Time); ok {
 		duration = time.Since(start)
 	}
 
-	requestInflight.WithLabelValues(request.Method, request.BaseUrl, request.Url).Dec()
-	requestCounter.WithLabelValues(request.Method, request.BaseUrl, request.Url, statusCode, errMsg).Inc()
-	requestDuration.WithLabelValues(request.Method, request.BaseUrl, request.Url, statusCode, errMsg).Observe(duration.Seconds())
+	url := stripQuery(request.Url)
+	requestInflight.WithLabelValues(request.Method, request.BaseUrl, url).Dec()
+	requestCounter.WithLabelValues(request.Method, request.BaseUrl, url, status, errMsg).Inc()
+	requestDuration.WithLabelValues(request.Method, request.BaseUrl, url, status, errMsg).Observe(duration.Seconds())
 
 	return nil
 }

pkg/client/redis/redis.go

@@ -52,8 +52,9 @@ func (m *Metric) Serve(context.Context) error {
 		defer ticker.Stop()
 
 		pusher := push.
-			New(m.cfg.PushTargetURL, config.ServiceName()+"-"+config.ServiceVersion()).
+			New(m.cfg.PushTargetURL, config.ServiceName()).
 			Gatherer(prometheus.DefaultRegisterer.(prometheus.Gatherer)).
+			Grouping("namespace", config.ServiceNamespace()).
 			Grouping("instance", system.Hostname())
 
 		for {