Emit metrics for requests to the Kubernetes control plane (#118)

* Rename file and type in preparation for adding metrics support

* Emit latency and result metrics for requests to the Kubernetes control plane

* Sort imports

cmd/rollout-operator/main.go

@@ -16,7 +16,6 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/tracing"
-	"github.com/opentracing-contrib/go-stdlib/nethttp"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -141,10 +140,7 @@ func main() {
 	// Build the Kubernetes client config.
 	kubeConfig, err := buildKubeConfig(cfg.kubeAPIURL, cfg.kubeConfigFile)
 	check(errors.Wrap(err, "failed to build Kubernetes client config"))
-
-	kubeConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
-		return instrumentation.NewKubernetesAPIClientTracer(&nethttp.Transport{RoundTripper: rt})
-	})
+	instrumentation.InstrumentKubernetesAPIClient(kubeConfig, reg)
 
 	kubeClient, err := kubernetes.NewForConfig(kubeConfig)
 	check(errors.Wrap(err, "failed to create Kubernetes client"))

pkg/instrumentation/kubernetes_api_client.go

@@ -0,0 +1,50 @@
+package instrumentation
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/grafana/dskit/instrument"
+	"github.com/opentracing-contrib/go-stdlib/nethttp"
+	"github.com/opentracing/opentracing-go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"k8s.io/client-go/rest"
+)
+
+type kubernetesAPIClientInstrumentation struct {
+	next http.RoundTripper
+	hist *prometheus.HistogramVec
+}
+
+func InstrumentKubernetesAPIClient(cfg *rest.Config, reg prometheus.Registerer) {
+	hist := promauto.With(reg).NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "rollout_operator_kubernetes_api_client_request_duration_seconds",
+			Help:    "Time (in seconds) spent waiting for requests to the Kubernetes API",
+			Buckets: instrument.DefBuckets,
+		},
+		[]string{"path", "method", "status_code"},
+	)
+
+	cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+		return &kubernetesAPIClientInstrumentation{
+			next: &nethttp.Transport{RoundTripper: rt},
+			hist: hist,
+		}
+	})
+}
+
+func (k *kubernetesAPIClientInstrumentation) RoundTrip(req *http.Request) (*http.Response, error) {
+	start := time.Now()
+
+	req, ht := nethttp.TraceRequest(opentracing.GlobalTracer(), req)
+	defer ht.Finish()
+
+	resp, err := k.next.RoundTrip(req)
+	duration := time.Since(start)
+	instrument.ObserveWithExemplar(req.Context(), k.hist.WithLabelValues(req.URL.EscapedPath(), req.Method, strconv.Itoa(resp.StatusCode)), duration.Seconds())
+
+	return resp, err
+}