Skip to content

Commit 1a0b139

Browse files
committed
instrument with otel metrics
1 parent 11859ff commit 1a0b139

File tree

8 files changed

+168
-14
lines changed

8 files changed

+168
-14
lines changed

controllers/redirect_controller.go

+35-3
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ package controllers
1818

1919
import (
2020
"context"
21+
"time"
2122

2223
"go.opentelemetry.io/otel/attribute"
24+
"go.opentelemetry.io/otel/metric/instrument"
2325
"go.opentelemetry.io/otel/trace"
2426
networkingv1 "k8s.io/api/networking/v1"
2527
k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -38,6 +40,20 @@ import (
3840
"github.com/prometheus/client_golang/prometheus"
3941
)
4042

43+
var redirects, _ = observability.Meter.Int64UpDownCounter(
44+
"urlshortener.active_redirects",
45+
instrument.WithUnit("count"),
46+
instrument.WithDescription("Amount of redirects (redirect one URL to another)"),
47+
)
48+
49+
var redirectCount int
50+
51+
var redirectReconcileLatency, _ = observability.Meter.Int64Histogram(
52+
"urlshortener.redirect_controller.reconcile_latency",
53+
instrument.WithUnit("microseconds"),
54+
instrument.WithDescription("How long does the reconcile function run for"),
55+
)
56+
4157
var activeRedirects = prometheus.NewGauge(
4258
prometheus.GaugeOpts{
4359
Name: "urlshortener_active_redirects",
@@ -82,15 +98,31 @@ func NewRedirectReconciler(client client.Client, rClient *redirectclient.Redirec
8298
//
8399
// For more details, check Reconcile and its Result here:
84100
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
85-
func (r *RedirectReconciler) Reconcile(c context.Context, req ctrl.Request) (ctrl.Result, error) {
86-
ctx, span := r.tracer.Start(c, "RedirectReconciler.Reconcile", trace.WithAttributes(attribute.String("redirect", req.Name)))
87-
defer span.End()
101+
func (r *RedirectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
102+
startTime := time.Now()
103+
104+
defer func() {
105+
redirectReconcileLatency.Record(ctx, time.Since(startTime).Microseconds(), attribute.String("redirect", req.NamespacedName.String()))
106+
}()
88107

89108
log := r.log.WithName("reconciler").WithValues("redirect", req.NamespacedName)
90109

110+
span := trace.SpanFromContext(ctx)
111+
112+
// Check if the span was sampled and is recording the data
113+
if !span.IsRecording() {
114+
ctx, span = r.tracer.Start(ctx, "RedirectReconciler.Reconcile")
115+
defer span.End()
116+
}
117+
118+
span.SetAttributes(attribute.String("redirect", req.Name))
119+
91120
// Monitor the number of redirects
92121
if redirectList, err := r.rClient.List(ctx); redirectList != nil && err == nil {
93122
activeRedirects.Set(float64(len(redirectList.Items)))
123+
124+
redirects.Add(ctx, int64(len(redirectList.Items)-redirectCount))
125+
redirectCount = len(redirectList.Items)
94126
}
95127

96128
// get Redirect from etcd

controllers/shortlink_controller.go

+35-4
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ package controllers
1818

1919
import (
2020
"context"
21+
"time"
2122

2223
"go.opentelemetry.io/otel/attribute"
24+
"go.opentelemetry.io/otel/metric/instrument"
2325
"go.opentelemetry.io/otel/trace"
2426

2527
"k8s.io/apimachinery/pkg/api/errors"
@@ -35,6 +37,20 @@ import (
3537
"github.com/prometheus/client_golang/prometheus"
3638
)
3739

40+
var shortlinks, _ = observability.Meter.Int64UpDownCounter(
41+
"urlshortener.active_shortlinks",
42+
instrument.WithUnit("count"),
43+
instrument.WithDescription("Amount of shortlinks (redirect a short-name to another URI)"),
44+
)
45+
46+
var shortlinkCount int
47+
48+
var shortlinkReconcileLatency, _ = observability.Meter.Int64Histogram(
49+
"urlshortener.shortlink_controller.reconcile_latency",
50+
instrument.WithUnit("microseconds"),
51+
instrument.WithDescription("How long does the reconcile function run for"),
52+
)
53+
3854
var activeShortlinks = prometheus.NewGauge(
3955
prometheus.GaugeOpts{
4056
Name: "urlshortener_active_shortlinks",
@@ -82,24 +98,39 @@ func NewShortLinkReconciler(client *shortlinkclient.ShortlinkClient, scheme *run
8298
//
8399
// For more details, check Reconcile and its Result here:
84100
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
85-
func (r *ShortLinkReconciler) Reconcile(c context.Context, req ctrl.Request) (ctrl.Result, error) {
86-
ctx, span := r.tracer.Start(c, "ShortLinkReconciler.Reconcile", trace.WithAttributes(attribute.String("shortlink", req.Name)))
87-
defer span.End()
101+
func (r *ShortLinkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
102+
startTime := time.Now()
103+
104+
defer func() {
105+
shortlinkReconcileLatency.Record(ctx, time.Since(startTime).Microseconds(), attribute.String("shortlink", req.NamespacedName.String()))
106+
}()
88107

89108
log := r.log.WithName("reconciler").WithValues("shortlink", req.NamespacedName.String())
90109

110+
span := trace.SpanFromContext(ctx)
111+
112+
// Check if the span was sampled and is recording the data
113+
if !span.IsRecording() {
114+
ctx, span = r.tracer.Start(ctx, "ShortLinkReconciler.Reconcile")
115+
defer span.End()
116+
}
117+
118+
span.SetAttributes(attribute.String("shortlink", req.NamespacedName.String()))
119+
91120
// Get ShortLink from etcd
92121
shortlink, err := r.client.GetNamespaced(ctx, req.NamespacedName)
93122
if err != nil || shortlink == nil {
94123
if errors.IsNotFound(err) {
95-
activeShortlinks.Dec()
96124
observability.RecordInfo(span, &log, "Shortlink resource not found. Ignoring since object must be deleted")
97125
} else {
98126
observability.RecordError(span, &log, err, "Failed to fetch ShortLink resource")
99127
}
100128
}
101129

102130
if shortlinkList, err := r.client.ListNamespaced(ctx, req.Namespace); shortlinkList != nil && err == nil {
131+
shortlinks.Add(ctx, int64(len(shortlinkList.Items)-shortlinkCount))
132+
shortlinkCount = len(shortlinkList.Items)
133+
103134
activeShortlinks.Set(float64(len(shortlinkList.Items)))
104135

105136
for _, shortlink := range shortlinkList.Items {

go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ require (
2929
require (
3030
github.com/felixge/httpsnoop v1.0.3 // indirect
3131
go.opentelemetry.io/otel/metric v0.37.0 // indirect
32+
go.opentelemetry.io/otel/sdk/metric v0.37.0 // indirect
3233
)
3334

3435
require (
@@ -84,6 +85,7 @@ require (
8485
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
8586
github.com/ugorji/go/codec v1.2.9 // indirect
8687
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
88+
go.opentelemetry.io/otel/exporters/prometheus v0.37.0
8789
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
8890
go.uber.org/atomic v1.10.0 // indirect
8991
go.uber.org/multierr v1.9.0 // indirect

go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -415,11 +415,15 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85
415415
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY=
416416
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.13.0 h1:Ntu7izEOIRHEgQNjbGc7j3eNtYMAiZfElJJ4JiiRDH4=
417417
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.13.0/go.mod h1:wZ9SAjm2sjw3vStBhlCfMZWZusyOQrwrHOFo00jyMC4=
418+
go.opentelemetry.io/otel/exporters/prometheus v0.37.0 h1:NQc0epfL0xItsmGgSXgfbH2C1fq2VLXkZoDFsfRNHpc=
419+
go.opentelemetry.io/otel/exporters/prometheus v0.37.0/go.mod h1:hB8qWjsStK36t50/R0V2ULFb4u95X/Q6zupXLgvjTh8=
418420
go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
419421
go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
420422
go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
421423
go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=
422424
go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM=
425+
go.opentelemetry.io/otel/sdk/metric v0.37.0 h1:haYBBtZZxiI3ROwSmkZnI+d0+AVzBWeviuYQDeBWosU=
426+
go.opentelemetry.io/otel/sdk/metric v0.37.0/go.mod h1:mO2WV1AZKKwhwHTV3AKOoIEb9LbUaENZDuGUQd+j4A0=
423427
go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU=
424428
go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
425429
go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=

main.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,19 @@ func main() {
109109
}
110110
}()
111111

112+
// Initialize Metrics (OpenTelemetry)
113+
meterProvider, err := observability.InitMetrics(serviceName, serviceVersion)
114+
if err != nil {
115+
setupLog.Error(err, "failed initializing tracing")
116+
os.Exit(1)
117+
}
118+
119+
defer func() {
120+
if err := meterProvider.Shutdown(context.Background()); err != nil {
121+
shutdownLog.Error(err, "Error shutting down metrics provider")
122+
}
123+
}()
124+
112125
// Start namespaced
113126
namespace := ""
114127

@@ -213,7 +226,7 @@ func main() {
213226

214227
// Init Gin Framework
215228
gin.SetMode(gin.ReleaseMode)
216-
r, srv := router.NewGinGonicHTTPServer(&setupLog, bindAddr)
229+
r, srv := router.NewGinGonicHTTPServer(&setupLog, bindAddr, serviceName)
217230

218231
setupLog.Info("Load API routes")
219232
router.Load(r, shortlinkController)

pkg/client/redirect_client.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package client
22

33
import (
44
"context"
5-
"io/ioutil"
65
"os"
76

87
"github.com/cedi/urlshortener/api/v1alpha1"
@@ -37,7 +36,7 @@ func (c *RedirectClient) Get(ct context.Context, name string) (*v1alpha1.Redirec
3736
defer span.End()
3837

3938
// try to read the namespace from /var/run
40-
namespace, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
39+
namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
4140
if err != nil {
4241
span.RecordError(err)
4342
return nil, errors.Wrap(err, "Unable to read current namespace")

pkg/observability/opentelemetry.go

+75-2
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,30 @@ package observability
22

33
import (
44
"context"
5+
"fmt"
56
"os"
67
"strings"
78

89
"github.com/MrAlias/flow"
910
"github.com/pkg/errors"
11+
"github.com/prometheus/client_golang/prometheus"
1012
"go.opentelemetry.io/otel"
1113
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
1214
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
15+
otelProm "go.opentelemetry.io/otel/exporters/prometheus"
16+
otelMetric "go.opentelemetry.io/otel/metric"
1317
"go.opentelemetry.io/otel/propagation"
18+
"go.opentelemetry.io/otel/sdk/metric"
1419
"go.opentelemetry.io/otel/sdk/resource"
1520
sdkTrace "go.opentelemetry.io/otel/sdk/trace"
1621
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
1722
"go.opentelemetry.io/otel/trace"
1823
)
1924

25+
var Meter otelMetric.Meter
26+
2027
func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, trace.Tracer, error) {
28+
ctx := context.Background()
2129

2230
otlpEndpoint, ok := os.LookupEnv("OTLP_ENDPOINT")
2331
otlpInsecure := os.Getenv("OTLP_INSECURE")
@@ -37,7 +45,7 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t
3745

3846
client := otlptracehttp.NewClient(otlpOptions...)
3947

40-
otlptracehttpExporter, err := otlptrace.New(context.Background(), client)
48+
otlptracehttpExporter, err := otlptrace.New(ctx, client)
4149
if err != nil {
4250
return nil, nil, errors.Wrap(err, "failed creating OTLP trace exporter")
4351
}
@@ -48,7 +56,7 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t
4856
}
4957

5058
resources, err := resource.New(
51-
context.Background(),
59+
ctx,
5260
resource.WithFromEnv(), // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables
5361
resource.WithOS(), // This option configures a set of Detectors that discover OS information
5462
resource.WithContainer(), // This option configures a set of Detectors that discover container information
@@ -80,3 +88,68 @@ func InitTracer(serviceName, serviceVersion string) (*sdkTrace.TracerProvider, t
8088

8189
return traceProvider, trace, nil
8290
}
91+
92+
func InitMetrics(serviceName, serviceVersion string) (*metric.MeterProvider, error) {
93+
ctx := context.Background()
94+
95+
otlpEndpoint, ok := os.LookupEnv("OTLP_ENDPOINT")
96+
otlpInsecure := os.Getenv("OTLP_INSECURE")
97+
98+
otlpOptions := make([]otlptracehttp.Option, 0)
99+
100+
if ok {
101+
otlpOptions = append(otlpOptions, otlptracehttp.WithEndpoint(otlpEndpoint))
102+
103+
if strings.ToLower(otlpInsecure) == "true" {
104+
otlpOptions = append(otlpOptions, otlptracehttp.WithInsecure())
105+
}
106+
} else {
107+
otlpOptions = append(otlpOptions, otlptracehttp.WithEndpoint("localhost:4318"))
108+
otlpOptions = append(otlpOptions, otlptracehttp.WithInsecure())
109+
}
110+
111+
registry := prometheus.NewRegistry()
112+
exporter, err := otelProm.New(
113+
otelProm.WithoutUnits(),
114+
otelProm.WithRegisterer(registry),
115+
)
116+
117+
if err != nil {
118+
return nil, err
119+
}
120+
121+
hostname, err := os.Hostname()
122+
if err != nil {
123+
return nil, err
124+
}
125+
126+
resources, err := resource.New(
127+
ctx,
128+
resource.WithFromEnv(), // pull attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables
129+
resource.WithOS(), // This option configures a set of Detectors that discover OS information
130+
resource.WithContainer(), // This option configures a set of Detectors that discover container information
131+
resource.WithHost(), // This option configures a set of Detectors that discover host information
132+
resource.WithAttributes(
133+
semconv.ServiceNameKey.String(serviceName),
134+
semconv.ServiceVersionKey.String(serviceVersion),
135+
semconv.ServiceInstanceIDKey.String(hostname),
136+
),
137+
)
138+
if err != nil {
139+
return nil, err
140+
}
141+
142+
resources, err = resource.Merge(resource.Default(), resources)
143+
if err != nil {
144+
return nil, err
145+
}
146+
147+
provider := metric.NewMeterProvider(
148+
metric.WithResource(resources),
149+
metric.WithReader(exporter),
150+
)
151+
152+
Meter = provider.Meter(fmt.Sprintf("%sMeter", serviceName))
153+
154+
return provider, nil
155+
}

pkg/router/router.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ import (
3333
// @in header
3434
// @name Authorization
3535

36-
func NewGinGonicHTTPServer(setupLog *logr.Logger, bindAddr string) (*gin.Engine, *http.Server) {
36+
func NewGinGonicHTTPServer(setupLog *logr.Logger, bindAddr, serviceName string) (*gin.Engine, *http.Server) {
3737
router := gin.New()
3838
router.Use(
39-
otelgin.Middleware("urlshortener"),
39+
otelgin.Middleware(serviceName),
4040
secure.Secure(secure.Options{
4141
SSLRedirect: true,
4242
SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},

0 commit comments

Comments
 (0)