diff --git a/bill-of-materials.json b/bill-of-materials.json index b495c09e337..16fd5131fd8 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -216,7 +216,16 @@ ] }, { - "project": "github.com/grpc-ecosystem/go-grpc-prometheus", + "project": "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, + { + "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", "licenses": [ { "type": "Apache License 2.0", @@ -785,6 +794,10 @@ { "project": "sigs.k8s.io/yaml", "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + }, { "type": "BSD 3-clause \"New\" or \"Revised\" License", "confidence": 1 diff --git a/bill-of-materials.override.json b/bill-of-materials.override.json index 15afc56409d..9d8d2e9bc3b 100644 --- a/bill-of-materials.override.json +++ b/bill-of-materials.override.json @@ -7,6 +7,15 @@ } ] }, + { + "project": "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, { "project": "github.com/inconshreveable/mousetrap", "licenses": [ diff --git a/client/v3/go.mod b/client/v3/go.mod index 939216ecb90..0aebfba0bd3 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.5 require ( github.com/coreos/go-semver v0.3.1 github.com/dustin/go-humanize v1.0.1 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/prometheus/client_golang v1.20.5 github.com/stretchr/testify v1.10.0 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 @@ -24,6 +24,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/client/v3/go.sum b/client/v3/go.sum index 2b02e26f8bf..82fce51ce70 100644 --- a/client/v3/go.sum +++ b/client/v3/go.sum @@ -24,8 +24,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/etcdctl/go.sum b/etcdctl/go.sum index 7e21083b798..956c83befba 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -32,8 +32,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/etcdutl/go.mod b/etcdutl/go.mod index d67a673d641..e758d1d08f4 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -52,7 +52,8 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/etcdutl/go.sum b/etcdutl/go.sum index fcb7222aec0..326f5180c6f 100644 --- a/etcdutl/go.sum +++ b/etcdutl/go.sum @@ -37,8 +37,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/go.mod b/go.mod index 026117dc96d..bad0af2a6dd 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect diff --git a/go.sum b/go.sum index dea496d1f80..77cd78df5c0 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/server/config/config.go b/server/config/config.go index 93bfb3362e3..986bde50e78 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -211,6 +211,9 @@ type ServerConfig struct { // ServerFeatureGate is a server level feature gate ServerFeatureGate featuregate.FeatureGate + + // Metrics types of metrics - should be either 'basic' or 'extensive' + Metrics string } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/etcd.go b/server/embed/etcd.go index bc9725171a6..5b3c3677beb 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -31,7 +31,6 @@ import ( "sync" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/soheilhy/cmux" "go.uber.org/zap" "google.golang.org/grpc" @@ -239,6 +238,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { V2Deprecation: cfg.V2DeprecationEffective(), ExperimentalLocalAddress: cfg.InferLocalAddr(), ServerFeatureGate: cfg.ServerFeatureGate, + Metrics: cfg.Metrics, } if srvcfg.ExperimentalEnableDistributedTracing { @@ -876,10 +876,6 @@ func (e *Etcd) createMetricsListener(murl url.URL) (net.Listener, error) { } func (e *Etcd) serveMetrics() (err error) { - if e.cfg.Metrics == "extensive" { - grpc_prometheus.EnableHandlingTimeHistogram() - } - if len(e.cfg.ListenMetricsUrls) > 0 { metricsMux := http.NewServeMux() etcdhttp.HandleMetrics(metricsMux) diff --git a/server/etcdmain/grpc_proxy.go b/server/etcdmain/grpc_proxy.go index a0df3f99ae4..9aeae162c20 100644 --- a/server/etcdmain/grpc_proxy.go +++ b/server/etcdmain/grpc_proxy.go @@ -30,8 +30,9 @@ import ( "time" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus" "github.com/soheilhy/cmux" "github.com/spf13/cobra" "go.uber.org/zap" @@ -505,11 +506,14 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server { alwaysLoggingDeciderServer := func(ctx context.Context, fullMethodName string, servingObject any) bool { return true } + serverMetrics := grpc_prometheus.NewServerMetrics() + prometheus.MustRegister(serverMetrics) + grpcChainStreamList := []grpc.StreamServerInterceptor{ - grpc_prometheus.StreamServerInterceptor, + serverMetrics.StreamServerInterceptor(), } grpcChainUnaryList := []grpc.UnaryServerInterceptor{ - grpc_prometheus.UnaryServerInterceptor, + serverMetrics.UnaryServerInterceptor(), } if grpcProxyEnableLogging { grpcChainStreamList = append(grpcChainStreamList, diff --git a/server/etcdserver/api/v3rpc/grpc.go b/server/etcdserver/api/v3rpc/grpc.go index 32949207805..810fd219b8f 100644 --- a/server/etcdserver/api/v3rpc/grpc.go +++ b/server/etcdserver/api/v3rpc/grpc.go @@ -18,8 +18,10 @@ import ( "crypto/tls" "math" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" @@ -39,10 +41,21 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer if tls != nil { opts = append(opts, grpc.Creds(credentials.NewTransportCredential(tls))) } + + var mopts []grpc_prometheus.ServerMetricsOption + if s.Cfg.Metrics == "extensive" { + mopts = append(mopts, grpc_prometheus.WithServerHandlingTimeHistogram()) + } + serverMetrics := grpc_prometheus.NewServerMetrics(mopts...) + err := prometheus.Register(serverMetrics) + if err != nil { + s.Cfg.Logger.Warn("etcdserver: failed to register grpc metrics", zap.Error(err)) + } + chainUnaryInterceptors := []grpc.UnaryServerInterceptor{ newLogUnaryInterceptor(s), newUnaryInterceptor(s), - grpc_prometheus.UnaryServerInterceptor, + serverMetrics.UnaryServerInterceptor(), } if interceptor != nil { chainUnaryInterceptors = append(chainUnaryInterceptors, interceptor) @@ -50,7 +63,7 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer chainStreamInterceptors := []grpc.StreamServerInterceptor{ newStreamInterceptor(s), - grpc_prometheus.StreamServerInterceptor, + serverMetrics.StreamServerInterceptor(), } if s.Cfg.ExperimentalEnableDistributedTracing { @@ -79,7 +92,7 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, interceptor grpc.UnarySer pb.RegisterMaintenanceServer(grpcServer, NewMaintenanceServer(s, healthNotifier)) // set zero values for metrics registered for this grpc server - grpc_prometheus.Register(grpcServer) + serverMetrics.InitializeMetrics(grpcServer) return grpcServer } diff --git a/server/etcdserver/api/v3rpc/metrics.go b/server/etcdserver/api/v3rpc/metrics.go index 9b432621ec5..d79506e0732 100644 --- a/server/etcdserver/api/v3rpc/metrics.go +++ b/server/etcdserver/api/v3rpc/metrics.go @@ -14,7 +14,9 @@ package v3rpc -import "github.com/prometheus/client_golang/prometheus" +import ( + "github.com/prometheus/client_golang/prometheus" +) var ( sentBytes = prometheus.NewCounter(prometheus.CounterOpts{ diff --git a/server/go.mod b/server/go.mod index e7e8c5a266e..40556abb845 100644 --- a/server/go.mod +++ b/server/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 github.com/jonboulle/clockwork v0.5.0 github.com/prometheus/client_golang v1.20.5 @@ -56,6 +56,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index 8b0fb813e8b..ed7ac3830d2 100644 --- a/server/go.sum +++ b/server/go.sum @@ -59,8 +59,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 10c718764b9..c81c10d0038 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -175,6 +175,7 @@ type ClusterConfig struct { MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration + Metrics string } type Cluster struct { @@ -292,6 +293,7 @@ func (c *Cluster) MustNewMember(t testutil.TB) *Member { MaxLearners: c.Cfg.MaxLearners, DisableStrictReconfigCheck: c.Cfg.DisableStrictReconfigCheck, CorruptCheckTime: c.Cfg.CorruptCheckTime, + Metrics: c.Cfg.Metrics, }) m.DiscoveryURL = c.Cfg.DiscoveryURL return m @@ -617,6 +619,7 @@ type MemberConfig struct { MaxLearners int DisableStrictReconfigCheck bool CorruptCheckTime time.Duration + Metrics string } // MustNewMember return an inited member with the given name. If peerTLS is @@ -731,6 +734,7 @@ func MustNewMember(t testutil.TB, mcfg MemberConfig) *Member { if mcfg.MaxLearners != 0 { m.MaxLearners = mcfg.MaxLearners } + m.Metrics = mcfg.Metrics m.V2Deprecation = config.V2_DEPR_DEFAULT m.GRPCServerRecorder = &grpctesting.GRPCRecorder{} diff --git a/tests/go.mod b/tests/go.mod index 2daab290525..492c177af0d 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -21,7 +21,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 @@ -71,6 +71,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/klauspost/compress v1.17.9 // indirect diff --git a/tests/go.sum b/tests/go.sum index 8cfcd1df8a7..8f13c0c4d0b 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -71,8 +71,10 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 h1:VD1gqscl4nYs1YxVuSdemTrSgTKrwOWDK0FVFMqm+Cg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0/go.mod h1:4EgsQoS4TOhJizV+JTFg40qx1Ofh3XmXEQNBpgvNT40= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/tests/integration/clientv3/examples/example_metrics_test.go b/tests/integration/clientv3/examples/example_metrics_test.go index d21c6d393e2..75b47e53d23 100644 --- a/tests/integration/clientv3/examples/example_metrics_test.go +++ b/tests/integration/clientv3/examples/example_metrics_test.go @@ -23,10 +23,12 @@ import ( "net/http" "strings" - grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - clientv3 "go.etcd.io/etcd/client/v3" "google.golang.org/grpc" + + clientv3 "go.etcd.io/etcd/client/v3" ) func mockClient_metrics() { @@ -35,11 +37,13 @@ func mockClient_metrics() { func ExampleClient_metrics() { forUnitTestsRunInMockedContext(mockClient_metrics, func() { + clientMetrics := grpcprom.NewClientMetrics() + prometheus.Register(clientMetrics) cli, err := clientv3.New(clientv3.Config{ Endpoints: exampleEndpoints(), DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), + grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, }) if err != nil { diff --git a/tests/integration/clientv3/metrics_test.go b/tests/integration/clientv3/metrics_test.go index 252ad6ec815..3188276c9c2 100644 --- a/tests/integration/clientv3/metrics_test.go +++ b/tests/integration/clientv3/metrics_test.go @@ -16,18 +16,18 @@ package clientv3test import ( "bufio" + "bytes" "context" "errors" "io" "net" "net/http" - "slices" "strconv" "strings" "testing" "time" - grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus" + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" @@ -75,11 +75,14 @@ func TestV3ClientMetrics(t *testing.T) { clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) defer clus.Terminate(t) + clientMetrics := grpcprom.NewClientMetrics() + prometheus.Register(clientMetrics) + cfg := clientv3.Config{ Endpoints: []string{clus.Members[0].GRPCURL}, DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor), + grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), + grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), }, } cli, cerr := integration2.NewClient(t, cfg) @@ -147,23 +150,9 @@ func sumCountersForMetricAndLabels(t *testing.T, url string, metricName string, } func getHTTPBodyAsLines(t *testing.T, url string) []string { - cfgtls := transport.TLSInfo{} - tr, err := transport.NewTransport(cfgtls, time.Second) - if err != nil { - t.Fatalf("Error getting transport: %v", err) - } - - tr.MaxIdleConns = -1 - tr.DisableKeepAlives = true - - cli := &http.Client{Transport: tr} - - resp, err := cli.Get(url) - if err != nil { - t.Fatalf("Error fetching: %v", err) - } + data := getHTTPBodyAsBytes(t, url) - reader := bufio.NewReader(resp.Body) + reader := bufio.NewReader(bytes.NewReader(data)) var lines []string for { line, err := reader.ReadString('\n') @@ -175,180 +164,29 @@ func getHTTPBodyAsLines(t *testing.T, url string) []string { } lines = append(lines, line) } - resp.Body.Close() return lines } -func TestAllMetricsGenerated(t *testing.T) { - integration2.BeforeTest(t) - - var ( - addr = "localhost:27989" - ln net.Listener - ) - - srv := &http.Server{Handler: promhttp.Handler()} - srv.SetKeepAlivesEnabled(false) - - ln, err := transport.NewUnixListener(addr) +func getHTTPBodyAsBytes(t *testing.T, url string) []byte { + cfgtls := transport.TLSInfo{} + tr, err := transport.NewTransport(cfgtls, time.Second) if err != nil { - t.Errorf("Error: %v occurred while listening on addr: %v", err, addr) + t.Fatalf("Error getting transport: %v", err) } - donec := make(chan struct{}) - defer func() { - ln.Close() - <-donec - }() - - // listen for all Prometheus metrics - go func() { - defer close(donec) - - serr := srv.Serve(ln) - if serr != nil && !transport.IsClosedConnError(serr) { - t.Errorf("Err serving http requests: %v", serr) - } - }() - - url := "unix://" + addr + "/metrics" - - clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) - defer clus.Terminate(t) - - clientMetrics := grpcprom.NewClientMetrics() - prometheus.Register(clientMetrics) + tr.MaxIdleConns = -1 + tr.DisableKeepAlives = true - cfg := clientv3.Config{ - Endpoints: []string{clus.Members[0].GRPCURL}, - DialOptions: []grpc.DialOption{ - grpc.WithUnaryInterceptor(clientMetrics.UnaryClientInterceptor()), - grpc.WithStreamInterceptor(clientMetrics.StreamClientInterceptor()), - }, - } - cli, cerr := integration2.NewClient(t, cfg) - if cerr != nil { - t.Fatal(cerr) - } - defer cli.Close() + cli := &http.Client{Transport: tr} - // Perform some operations to generate metrics - wc := cli.Watch(context.Background(), "foo") - _, err = cli.Put(context.Background(), "foo", "bar") + resp, err := cli.Get(url) if err != nil { - t.Errorf("Error putting value in key store") - } - - // consume watch response - select { - case <-wc: - case <-time.After(10 * time.Second): - t.Error("Timeout occurred for getting watch response") - } - - // Define the expected list of metrics - expectedMetrics := []string{ - "etcd_cluster_version", - "etcd_disk_backend_commit_duration_seconds_bucket", - "etcd_disk_backend_commit_duration_seconds_count", - "etcd_disk_backend_commit_duration_seconds_sum", - "etcd_disk_backend_defrag_duration_seconds_bucket", - "etcd_disk_backend_defrag_duration_seconds_count", - "etcd_disk_backend_defrag_duration_seconds_sum", - "etcd_disk_backend_snapshot_duration_seconds_bucket", - "etcd_disk_backend_snapshot_duration_seconds_count", - "etcd_disk_backend_snapshot_duration_seconds_sum", - "etcd_disk_defrag_inflight", - "etcd_disk_wal_fsync_duration_seconds_bucket", - "etcd_disk_wal_fsync_duration_seconds_count", - "etcd_disk_wal_fsync_duration_seconds_sum", - "etcd_disk_wal_write_bytes_total", - "etcd_disk_wal_write_duration_seconds_bucket", - "etcd_disk_wal_write_duration_seconds_count", - "etcd_disk_wal_write_duration_seconds_sum", - "etcd_mvcc_db_open_read_transactions", - "etcd_mvcc_db_total_size_in_bytes", - "etcd_mvcc_db_total_size_in_use_in_bytes", - "etcd_mvcc_delete_total", - "etcd_mvcc_hash_duration_seconds_bucket", - "etcd_mvcc_hash_duration_seconds_count", - "etcd_mvcc_hash_duration_seconds_sum", - "etcd_mvcc_hash_rev_duration_seconds_bucket", - "etcd_mvcc_hash_rev_duration_seconds_count", - "etcd_mvcc_hash_rev_duration_seconds_sum", - "etcd_mvcc_put_total", - "etcd_mvcc_range_total", - "etcd_mvcc_txn_total", - "etcd_network_client_grpc_received_bytes_total", - "etcd_network_client_grpc_sent_bytes_total", - "etcd_network_known_peers", - "etcd_server_apply_duration_seconds_bucket", - "etcd_server_apply_duration_seconds_count", - "etcd_server_apply_duration_seconds_sum", - "etcd_server_client_requests_total", - "etcd_server_go_version", - "etcd_server_has_leader", - "etcd_server_health_failures", - "etcd_server_health_success", - "etcd_server_heartbeat_send_failures_total", - "etcd_server_id", - "etcd_server_is_leader", - "etcd_server_is_learner", - "etcd_server_leader_changes_seen_total", - "etcd_server_learner_promote_successes", - "etcd_server_proposals_applied_total", - "etcd_server_proposals_committed_total", - "etcd_server_proposals_failed_total", - "etcd_server_proposals_pending", - "etcd_server_quota_backend_bytes", - "etcd_server_read_indexes_failed_total", - "etcd_server_slow_apply_total", - "etcd_server_slow_read_indexes_total", - "etcd_server_snapshot_apply_in_progress_total", - "etcd_server_version", - "etcd_snap_db_fsync_duration_seconds_bucket", - "etcd_snap_db_fsync_duration_seconds_count", - "etcd_snap_db_fsync_duration_seconds_sum", - "etcd_snap_db_save_total_duration_seconds_bucket", - "etcd_snap_db_save_total_duration_seconds_count", - "etcd_snap_db_save_total_duration_seconds_sum", - "etcd_snap_fsync_duration_seconds_bucket", - "etcd_snap_fsync_duration_seconds_count", - "etcd_snap_fsync_duration_seconds_sum", - "grpc_client_handled_total", - "grpc_client_msg_received_total", - "grpc_client_msg_sent_total", - "grpc_client_started_total", - "grpc_server_handled_total", - "grpc_server_msg_received_total", - "grpc_server_msg_sent_total", - "grpc_server_started_total", - } - - // Get the list of generated metrics - generatedMetrics := getMetricsList(t, url) - for _, metric := range expectedMetrics { - if !slices.Contains(generatedMetrics, metric) { - t.Errorf("Expected metric %s not found in generated metrics", metric) - } - } -} - -func getMetricsList(t *testing.T, url string) []string { - lines := getHTTPBodyAsLines(t, url) - metrics := make(map[string]struct{}) - for _, line := range lines { - if strings.Contains(line, "{") { - metric := line[:strings.Index(line, "{")] - metrics[metric] = struct{}{} - } else { - metric := line[:strings.Index(line, " ")] - metrics[metric] = struct{}{} - } + t.Fatalf("Error fetching: %v", err) } - var metricList []string - for metric := range metrics { - metricList = append(metricList, metric) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Error reading http body: %v", err) } - return metricList + return body }