diff --git a/controllers/redirect_controller.go b/controllers/redirect_controller.go index e05ef0e..a9cfb0f 100644 --- a/controllers/redirect_controller.go +++ b/controllers/redirect_controller.go @@ -18,8 +18,11 @@ package controllers import ( "context" + "time" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/trace" networkingv1 "k8s.io/api/networking/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -38,6 +41,20 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +var redirects, _ = observability.Meter.Int64UpDownCounter( + "urlshortener.active_redirects", + instrument.WithUnit("count"), + instrument.WithDescription("Amount of redirects (redirect one URL to another)"), +) + +var redirectCount int + +var redirectReconcileLatency, _ = observability.Meter.Int64Histogram( + "urlshortener.redirect_controller.reconcile_latency", + instrument.WithUnit("microseconds"), + instrument.WithDescription("How long does the reconcile function run for"), +) + var activeRedirects = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "urlshortener_active_redirects", @@ -57,16 +74,18 @@ type RedirectReconciler struct { scheme *runtime.Scheme log *logr.Logger tracer trace.Tracer + meter metric.Meter } // NewRedirectReconciler returns a new RedirectReconciler -func NewRedirectReconciler(client client.Client, rClient *redirectclient.RedirectClient, scheme *runtime.Scheme, log *logr.Logger, tracer trace.Tracer) *RedirectReconciler { +func NewRedirectReconciler(client client.Client, rClient *redirectclient.RedirectClient, scheme *runtime.Scheme, log *logr.Logger, tracer trace.Tracer, meter metric.Meter) *RedirectReconciler { return &RedirectReconciler{ client: client, rClient: rClient, scheme: scheme, log: log, tracer: tracer, + meter: meter, } } @@ -82,15 +101,31 @@ func NewRedirectReconciler(client client.Client, rClient *redirectclient.Redirec // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile -func (r *RedirectReconciler) Reconcile(c context.Context, req ctrl.Request) (ctrl.Result, error) { - ctx, span := r.tracer.Start(c, "RedirectReconciler.Reconcile", trace.WithAttributes(attribute.String("redirect", req.Name))) - defer span.End() +func (r *RedirectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + startTime := time.Now() + + defer func() { + redirectReconcileLatency.Record(ctx, time.Since(startTime).Microseconds(), attribute.String("redirect", req.NamespacedName.String())) + }() log := r.log.WithName("reconciler").WithValues("redirect", req.NamespacedName) + span := trace.SpanFromContext(ctx) + + // Check if the span was sampled and is recording the data + if !span.IsRecording() { + ctx, span = r.tracer.Start(ctx, "RedirectReconciler.Reconcile") + defer span.End() + } + + span.SetAttributes(attribute.String("redirect", req.Name)) + // Monitor the number of redirects if redirectList, err := r.rClient.List(ctx); redirectList != nil && err == nil { activeRedirects.Set(float64(len(redirectList.Items))) + + redirects.Add(ctx, int64(len(redirectList.Items)-redirectCount)) + redirectCount = len(redirectList.Items) } // get Redirect from etcd diff --git a/controllers/shortlink_controller.go b/controllers/shortlink_controller.go index b072b4e..46ba477 100644 --- a/controllers/shortlink_controller.go +++ b/controllers/shortlink_controller.go @@ -18,8 +18,11 @@ package controllers import ( "context" + "time" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/trace" "k8s.io/apimachinery/pkg/api/errors" @@ -35,6 +38,20 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +var shortlinks, _ = observability.Meter.Int64UpDownCounter( + "urlshortener.active_shortlinks", + instrument.WithUnit("count"), + instrument.WithDescription("Amount of shortlinks (redirect a short-name to another URI)"), +) + +var shortlinkCount int + +var shortlinkReconcileLatency, _ = observability.Meter.Int64Histogram( + "urlshortener.shortlink_controller.reconcile_latency", + instrument.WithUnit("microseconds"), + instrument.WithDescription("How long does the reconcile function run for"), +) + var activeShortlinks = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "urlshortener_active_shortlinks", @@ -61,15 +78,17 @@ type ShortLinkReconciler struct { scheme *runtime.Scheme log *logr.Logger tracer trace.Tracer + meter metric.Meter } // NewShortLinkReconciler returns a new ShortLinkReconciler -func NewShortLinkReconciler(client *shortlinkclient.ShortlinkClient, scheme *runtime.Scheme, log *logr.Logger, tracer trace.Tracer) *ShortLinkReconciler { +func NewShortLinkReconciler(client *shortlinkclient.ShortlinkClient, scheme *runtime.Scheme, log *logr.Logger, tracer trace.Tracer, meter metric.Meter) *ShortLinkReconciler { return &ShortLinkReconciler{ client: client, scheme: scheme, log: log, tracer: tracer, + meter: meter, } } @@ -82,17 +101,29 @@ func NewShortLinkReconciler(client *shortlinkclient.ShortlinkClient, scheme *run // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile -func (r *ShortLinkReconciler) Reconcile(c context.Context, req ctrl.Request) (ctrl.Result, error) { - ctx, span := r.tracer.Start(c, "ShortLinkReconciler.Reconcile", trace.WithAttributes(attribute.String("shortlink", req.Name))) - defer span.End() +func (r *ShortLinkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + startTime := time.Now() + + defer func() { + shortlinkReconcileLatency.Record(ctx, time.Since(startTime).Microseconds(), attribute.String("shortlink", req.NamespacedName.String())) + }() log := r.log.WithName("reconciler").WithValues("shortlink", req.NamespacedName.String()) + span := trace.SpanFromContext(ctx) + + // Check if the span was sampled and is recording the data + if !span.IsRecording() { + ctx, span = r.tracer.Start(ctx, "ShortLinkReconciler.Reconcile") + defer span.End() + } + + span.SetAttributes(attribute.String("shortlink", req.NamespacedName.String())) + // Get ShortLink from etcd shortlink, err := r.client.GetNamespaced(ctx, req.NamespacedName) if err != nil || shortlink == nil { if errors.IsNotFound(err) { - activeShortlinks.Dec() observability.RecordInfo(span, &log, "Shortlink resource not found. Ignoring since object must be deleted") } else { observability.RecordError(span, &log, err, "Failed to fetch ShortLink resource") @@ -100,6 +131,9 @@ func (r *ShortLinkReconciler) Reconcile(c context.Context, req ctrl.Request) (ct } if shortlinkList, err := r.client.ListNamespaced(ctx, req.Namespace); shortlinkList != nil && err == nil { + shortlinks.Add(ctx, int64(len(shortlinkList.Items)-shortlinkCount)) + shortlinkCount = len(shortlinkList.Items) + activeShortlinks.Set(float64(len(shortlinkList.Items))) for _, shortlink := range shortlinkList.Items { diff --git a/go.mod b/go.mod index 6a0f4c3..f7516aa 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( require ( github.com/felixge/httpsnoop v1.0.3 // indirect go.opentelemetry.io/otel/metric v0.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v0.37.0 // indirect ) require ( @@ -84,6 +85,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.9 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.37.0 go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect diff --git a/go.sum b/go.sum index 2dfbeaf..df3fa6b 100644 --- a/go.sum +++ b/go.sum @@ -415,11 +415,15 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.13.0 h1:Ntu7izEOIRHEgQNjbGc7j3eNtYMAiZfElJJ4JiiRDH4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.13.0/go.mod h1:wZ9SAjm2sjw3vStBhlCfMZWZusyOQrwrHOFo00jyMC4= +go.opentelemetry.io/otel/exporters/prometheus v0.37.0 h1:NQc0epfL0xItsmGgSXgfbH2C1fq2VLXkZoDFsfRNHpc= +go.opentelemetry.io/otel/exporters/prometheus v0.37.0/go.mod h1:hB8qWjsStK36t50/R0V2ULFb4u95X/Q6zupXLgvjTh8= go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs= go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= +go.opentelemetry.io/otel/sdk/metric v0.37.0 h1:haYBBtZZxiI3ROwSmkZnI+d0+AVzBWeviuYQDeBWosU= +go.opentelemetry.io/otel/sdk/metric v0.37.0/go.mod h1:mO2WV1AZKKwhwHTV3AKOoIEb9LbUaENZDuGUQd+j4A0= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= diff --git a/main.go b/main.go index 538c63a..6e31356 100644 --- a/main.go +++ b/main.go @@ -109,6 +109,19 @@ func main() { } }() + // Initialize Metrics (OpenTelemetry) + meterProvider, err := observability.InitMetrics(serviceName, serviceVersion) + if err != nil { + setupLog.Error(err, "failed initializing tracing") + os.Exit(1) + } + + defer func() { + if err := meterProvider.Shutdown(context.Background()); err != nil { + shutdownLog.Error(err, "Error shutting down metrics provider") + } + }() + // Start namespaced namespace := "" @@ -213,7 +226,7 @@ func main() { // Init Gin Framework gin.SetMode(gin.ReleaseMode) - r, srv := router.NewGinGonicHTTPServer(&setupLog, bindAddr) + r, srv := router.NewGinGonicHTTPServer(&setupLog, bindAddr, serviceName) setupLog.Info("Load API routes") router.Load(r, shortlinkController) diff --git a/pkg/client/redirect_client.go b/pkg/client/redirect_client.go index c203451..d69f9e1 100644 --- a/pkg/client/redirect_client.go +++ b/pkg/client/redirect_client.go @@ -2,7 +2,6 @@ package client import ( "context" - "io/ioutil" "os" "github.com/cedi/urlshortener/api/v1alpha1" @@ -37,7 +36,7 @@ func (c *RedirectClient) Get(ct context.Context, name string) (*v1alpha1.Redirec defer span.End() // try to read the namespace from /var/run - namespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") if err != nil { span.RecordError(err) return nil, errors.Wrap(err, "Unable to read current namespace") diff --git a/pkg/observability/opentelemetry.go b/pkg/observability/opentelemetry.go index 86e83c8..5fc967f 100644 --- a/pkg/observability/opentelemetry.go +++ b/pkg/observability/opentelemetry.go @@ -2,22 +2,30 @@ package observability import ( "context" + "fmt" "os" "strings" "github.com/MrAlias/flow" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + otelProm "go.opentelemetry.io/otel/exporters/prometheus" + otelMetric "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" sdkTrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "go.opentelemetry.io/otel/trace" ) +var Meter otelMetric.Meter + func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, trace.Tracer, error) { + ctx := context.Background() otlpEndpoint, ok := os.LookupEnv("OTLP_ENDPOINT") otlpInsecure := os.Getenv("OTLP_INSECURE") @@ -37,7 +45,7 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t client := otlptracehttp.NewClient(otlpOptions...) - otlptracehttpExporter, err := otlptrace.New(context.Background(), client) + otlptracehttpExporter, err := otlptrace.New(ctx, client) if err != nil { return nil, nil, errors.Wrap(err, "failed creating OTLP trace exporter") } @@ -48,7 +56,7 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t } resources, err := resource.New( - context.Background(), + ctx, resource.WithFromEnv(), // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables resource.WithOS(), // This option configures a set of Detectors that discover OS information resource.WithContainer(), // This option configures a set of Detectors that discover container information @@ -80,3 +88,71 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t return traceProvider, trace, nil } + +func InitMetrics(serviceName, serviceVersion string) (*metric.MeterProvider, error) { + ctx := context.Background() + + otlpEndpoint, ok := os.LookupEnv("OTLP_ENDPOINT") + otlpInsecure := os.Getenv("OTLP_INSECURE") + + otlpOptions := make([]otlptracehttp.Option, 0) + + if ok { + otlpOptions = append(otlpOptions, otlptracehttp.WithEndpoint(otlpEndpoint)) + + if strings.ToLower(otlpInsecure) == "true" { + otlpOptions = append(otlpOptions, otlptracehttp.WithInsecure()) + } + } else { + otlpOptions = append(otlpOptions, otlptracehttp.WithEndpoint("localhost:4318")) + otlpOptions = append(otlpOptions, otlptracehttp.WithInsecure()) + } + + registry := prometheus.NewRegistry() + exporter, err := otelProm.New( + otelProm.WithoutUnits(), + otelProm.WithRegisterer(registry), + ) + + if err != nil { + return nil, err + } + + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + + resources, err := resource.New( + ctx, + resource.WithFromEnv(), // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables + resource.WithOS(), // This option configures a set of Detectors that discover OS information + resource.WithContainer(), // This option configures a set of Detectors that discover container information + resource.WithHost(), // This option configures a set of Detectors that discover host information + resource.WithAttributes( + semconv.ServiceNameKey.String(serviceName), + semconv.ServiceVersionKey.String(serviceVersion), + semconv.ServiceInstanceIDKey.String(hostname), + ), + ) + if err != nil { + return nil, err + } + + resources, err = resource.Merge(resource.Default(), resources) + if err != nil { + return nil, err + } + + provider := metric.NewMeterProvider( + metric.WithResource(resources), + metric.WithReader(exporter), + metric.Instrument{ + Name: "*", + }, + ) + + Meter = provider.Meter(fmt.Sprintf("%sMeter", serviceName)) + + return provider, nil +} diff --git a/pkg/router/router.go b/pkg/router/router.go index 64657b2..03c6a7d 100644 --- a/pkg/router/router.go +++ b/pkg/router/router.go @@ -33,10 +33,10 @@ import ( // @in header // @name Authorization -func NewGinGonicHTTPServer(setupLog *logr.Logger, bindAddr string) (*gin.Engine, *http.Server) { +func NewGinGonicHTTPServer(setupLog *logr.Logger, bindAddr, serviceName string) (*gin.Engine, *http.Server) { router := gin.New() router.Use( - otelgin.Middleware("urlshortener"), + otelgin.Middleware(serviceName), secure.Secure(secure.Options{ SSLRedirect: true, SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},