instrument with otel metrics

controllers/redirect_controller.go

@@ -18,8 +18,10 @@ package controllers
 
 import (
 	"context"
+	"time"
 
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/trace"
 	networkingv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -38,6 +40,20 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+var redirects, _ = observability.Meter.Int64UpDownCounter(
+	"urlshortener.active_redirects",
+	instrument.WithUnit("count"),
+	instrument.WithDescription("Amount of redirects (redirect one URL to another)"),
+)
+
+var redirectCount int
+
+var redirectReconcileLatency, _ = observability.Meter.Int64Histogram(
+	"urlshortener.redirect_controller.reconcile_latency",
+	instrument.WithUnit("microseconds"),
+	instrument.WithDescription("How long does the reconcile function run for"),
+)
+
 var activeRedirects = prometheus.NewGauge(
 	prometheus.GaugeOpts{
 		Name: "urlshortener_active_redirects",
@@ -82,15 +98,31 @@ func NewRedirectReconciler(client client.Client, rClient *redirectclient.Redirec
 //
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
-func (r *RedirectReconciler) Reconcile(c context.Context, req ctrl.Request) (ctrl.Result, error) {
-	ctx, span := r.tracer.Start(c, "RedirectReconciler.Reconcile", trace.WithAttributes(attribute.String("redirect", req.Name)))
-	defer span.End()
+func (r *RedirectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	startTime := time.Now()
+
+	defer func() {
+		redirectReconcileLatency.Record(ctx, time.Since(startTime).Microseconds(), attribute.String("redirect", req.NamespacedName.String()))
+	}()
 
 	log := r.log.WithName("reconciler").WithValues("redirect", req.NamespacedName)
 
+	span := trace.SpanFromContext(ctx)
+
+	// Check if the span was sampled and is recording the data
+	if !span.IsRecording() {
+		ctx, span = r.tracer.Start(ctx, "RedirectReconciler.Reconcile")
+		defer span.End()
+	}
+
+	span.SetAttributes(attribute.String("redirect", req.Name))
+
 	// Monitor the number of redirects
 	if redirectList, err := r.rClient.List(ctx); redirectList != nil && err == nil {
 		activeRedirects.Set(float64(len(redirectList.Items)))
+
+		redirects.Add(ctx, int64(len(redirectList.Items)-redirectCount))
+		redirectCount = len(redirectList.Items)
 	}
 
 	// get Redirect from etcd

controllers/shortlink_controller.go

@@ -18,8 +18,10 @@ package controllers
 
 import (
 	"context"
+	"time"
 
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/trace"
 
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -35,6 +37,20 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+var shortlinks, _ = observability.Meter.Int64UpDownCounter(
+	"urlshortener.active_shortlinks",
+	instrument.WithUnit("count"),
+	instrument.WithDescription("Amount of shortlinks (redirect a short-name to another URI)"),
+)
+
+var shortlinkCount int
+
+var shortlinkReconcileLatency, _ = observability.Meter.Int64Histogram(
+	"urlshortener.shortlink_controller.reconcile_latency",
+	instrument.WithUnit("microseconds"),
+	instrument.WithDescription("How long does the reconcile function run for"),
+)
+
 var activeShortlinks = prometheus.NewGauge(
 	prometheus.GaugeOpts{
 		Name: "urlshortener_active_shortlinks",
@@ -82,24 +98,39 @@ func NewShortLinkReconciler(client *shortlinkclient.ShortlinkClient, scheme *run
 //
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
-func (r *ShortLinkReconciler) Reconcile(c context.Context, req ctrl.Request) (ctrl.Result, error) {
-	ctx, span := r.tracer.Start(c, "ShortLinkReconciler.Reconcile", trace.WithAttributes(attribute.String("shortlink", req.Name)))
-	defer span.End()
+func (r *ShortLinkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	startTime := time.Now()
+
+	defer func() {
+		shortlinkReconcileLatency.Record(ctx, time.Since(startTime).Microseconds(), attribute.String("shortlink", req.NamespacedName.String()))
+	}()
 
 	log := r.log.WithName("reconciler").WithValues("shortlink", req.NamespacedName.String())
 
+	span := trace.SpanFromContext(ctx)
+
+	// Check if the span was sampled and is recording the data
+	if !span.IsRecording() {
+		ctx, span = r.tracer.Start(ctx, "ShortLinkReconciler.Reconcile")
+		defer span.End()
+	}
+
+	span.SetAttributes(attribute.String("shortlink", req.NamespacedName.String()))
+
 	// Get ShortLink from etcd
 	shortlink, err := r.client.GetNamespaced(ctx, req.NamespacedName)
 	if err != nil || shortlink == nil {
 		if errors.IsNotFound(err) {
-			activeShortlinks.Dec()
 			observability.RecordInfo(span, &log, "Shortlink resource not found. Ignoring since object must be deleted")
 		} else {
 			observability.RecordError(span, &log, err, "Failed to fetch ShortLink resource")
 		}
 	}
 
 	if shortlinkList, err := r.client.ListNamespaced(ctx, req.Namespace); shortlinkList != nil && err == nil {
+		shortlinks.Add(ctx, int64(len(shortlinkList.Items)-shortlinkCount))
+		shortlinkCount = len(shortlinkList.Items)
+
 		activeShortlinks.Set(float64(len(shortlinkList.Items)))
 
 		for _, shortlink := range shortlinkList.Items {

go.mod

@@ -29,6 +29,7 @@ require (
 require (
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	go.opentelemetry.io/otel/metric v0.37.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v0.37.0 // indirect
 )
 
 require (
@@ -84,6 +85,7 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.9 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.37.0
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect

go.sum

@@ -415,11 +415,15 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.13.0 h1:Ntu7izEOIRHEgQNjbGc7j3eNtYMAiZfElJJ4JiiRDH4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.13.0/go.mod h1:wZ9SAjm2sjw3vStBhlCfMZWZusyOQrwrHOFo00jyMC4=
+go.opentelemetry.io/otel/exporters/prometheus v0.37.0 h1:NQc0epfL0xItsmGgSXgfbH2C1fq2VLXkZoDFsfRNHpc=
+go.opentelemetry.io/otel/exporters/prometheus v0.37.0/go.mod h1:hB8qWjsStK36t50/R0V2ULFb4u95X/Q6zupXLgvjTh8=
 go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
 go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
 go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
 go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
 go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
+go.opentelemetry.io/otel/sdk/metric v0.37.0 h1:haYBBtZZxiI3ROwSmkZnI+d0+AVzBWeviuYQDeBWosU=
+go.opentelemetry.io/otel/sdk/metric v0.37.0/go.mod h1:mO2WV1AZKKwhwHTV3AKOoIEb9LbUaENZDuGUQd+j4A0=
 go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU=
 go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
 go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=

main.go

@@ -109,6 +109,19 @@ func main() {
 		}
 	}()
 
+	// Initialize Metrics (OpenTelemetry)
+	meterProvider, err := observability.InitMetrics(serviceName, serviceVersion)
+	if err != nil {
+		setupLog.Error(err, "failed initializing tracing")
+		os.Exit(1)
+	}
+
+	defer func() {
+		if err := meterProvider.Shutdown(context.Background()); err != nil {
+			shutdownLog.Error(err, "Error shutting down metrics provider")
+		}
+	}()
+
 	// Start namespaced
 	namespace := ""
 
@@ -213,7 +226,7 @@ func main() {
 
 	// Init Gin Framework
 	gin.SetMode(gin.ReleaseMode)
-	r, srv := router.NewGinGonicHTTPServer(&setupLog, bindAddr)
+	r, srv := router.NewGinGonicHTTPServer(&setupLog, bindAddr, serviceName)
 
 	setupLog.Info("Load API routes")
 	router.Load(r, shortlinkController)

pkg/client/redirect_client.go

@@ -2,7 +2,6 @@ package client
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 
 	"github.com/cedi/urlshortener/api/v1alpha1"
@@ -37,7 +36,7 @@ func (c *RedirectClient) Get(ct context.Context, name string) (*v1alpha1.Redirec
 	defer span.End()
 
 	// try to read the namespace from /var/run
-	namespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 	if err != nil {
 		span.RecordError(err)
 		return nil, errors.Wrap(err, "Unable to read current namespace")

pkg/observability/opentelemetry.go

@@ -2,22 +2,30 @@ package observability
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"strings"
 
 	"github.com/MrAlias/flow"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	otelProm "go.opentelemetry.io/otel/exporters/prometheus"
+	otelMetric "go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdkTrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 	"go.opentelemetry.io/otel/trace"
 )
 
+var Meter otelMetric.Meter
+
 func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, trace.Tracer, error) {
+	ctx := context.Background()
 
 	otlpEndpoint, ok := os.LookupEnv("OTLP_ENDPOINT")
 	otlpInsecure := os.Getenv("OTLP_INSECURE")
@@ -37,7 +45,7 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t
 
 	client := otlptracehttp.NewClient(otlpOptions...)
 
-	otlptracehttpExporter, err := otlptrace.New(context.Background(), client)
+	otlptracehttpExporter, err := otlptrace.New(ctx, client)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed creating OTLP trace exporter")
 	}
@@ -48,7 +56,7 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t
 	}
 
 	resources, err := resource.New(
-		context.Background(),
+		ctx,
 		resource.WithFromEnv(),   // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables
 		resource.WithOS(),        // This option configures a set of Detectors that discover OS information
 		resource.WithContainer(), // This option configures a set of Detectors that discover container information
@@ -80,3 +88,68 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t
 
 	return traceProvider, trace, nil
 }
+
+func InitMetrics(serviceName, serviceVersion string) (*metric.MeterProvider, error) {
+	ctx := context.Background()
+
+	otlpEndpoint, ok := os.LookupEnv("OTLP_ENDPOINT")
+	otlpInsecure := os.Getenv("OTLP_INSECURE")
+
+	otlpOptions := make([]otlptracehttp.Option, 0)
+
+	if ok {
+		otlpOptions = append(otlpOptions, otlptracehttp.WithEndpoint(otlpEndpoint))
+
+		if strings.ToLower(otlpInsecure) == "true" {
+			otlpOptions = append(otlpOptions, otlptracehttp.WithInsecure())
+		}
+	} else {
+		otlpOptions = append(otlpOptions, otlptracehttp.WithEndpoint("localhost:4318"))
+		otlpOptions = append(otlpOptions, otlptracehttp.WithInsecure())
+	}
+
+	registry := prometheus.NewRegistry()
+	exporter, err := otelProm.New(
+		otelProm.WithoutUnits(),
+		otelProm.WithRegisterer(registry),
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+
+	resources, err := resource.New(
+		ctx,
+		resource.WithFromEnv(),   // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables
+		resource.WithOS(),        // This option configures a set of Detectors that discover OS information
+		resource.WithContainer(), // This option configures a set of Detectors that discover container information
+		resource.WithHost(),      // This option configures a set of Detectors that discover host information
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(serviceName),
+			semconv.ServiceVersionKey.String(serviceVersion),
+			semconv.ServiceInstanceIDKey.String(hostname),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	resources, err = resource.Merge(resource.Default(), resources)
+	if err != nil {
+		return nil, err
+	}
+
+	provider := metric.NewMeterProvider(
+		metric.WithResource(resources),
+		metric.WithReader(exporter),
+	)
+
+	Meter = provider.Meter(fmt.Sprintf("%sMeter", serviceName))
+
+	return provider, nil
+}

pkg/router/router.go

@@ -33,10 +33,10 @@ import (
 // @in header
 // @name Authorization
 
-func NewGinGonicHTTPServer(setupLog *logr.Logger, bindAddr string) (*gin.Engine, *http.Server) {
+func NewGinGonicHTTPServer(setupLog *logr.Logger, bindAddr, serviceName string) (*gin.Engine, *http.Server) {
 	router := gin.New()
 	router.Use(
-		otelgin.Middleware("urlshortener"),
+		otelgin.Middleware(serviceName),
 		secure.Secure(secure.Options{
 			SSLRedirect:           true,
 			SSLProxyHeaders:       map[string]string{"X-Forwarded-Proto": "https"},