From 1fe770dc37d83f8f332347696e6797cff4fefdeb Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Fri, 17 May 2024 13:54:32 -0700 Subject: [PATCH 1/4] Pool resource log maps --- .../otlplog/otlploghttp/internal/transform/log.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go index a59536aa704..79af903542f 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go @@ -6,6 +6,7 @@ package transform // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/transform" import ( + "sync" "time" cpb "go.opentelemetry.io/proto/otlp/common/v1" @@ -25,6 +26,11 @@ func ResourceLogs(records []log.Record) []*lpb.ResourceLogs { } resMap := resourceLogsMap(records) + defer func() { + clear(resMap) + resourceLogsMapPool.Put(resMap) + }() + out := make([]*lpb.ResourceLogs, 0, len(resMap)) for _, rl := range resMap { out = append(out, rl) @@ -32,8 +38,14 @@ func ResourceLogs(records []log.Record) []*lpb.ResourceLogs { return out } +var resourceLogsMapPool = sync.Pool{ + New: func() any { + return make(map[attribute.Distinct]*lpb.ResourceLogs) + }, +} + func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceLogs { - out := make(map[attribute.Distinct]*lpb.ResourceLogs) + out := resourceLogsMapPool.Get().(map[attribute.Distinct]*lpb.ResourceLogs) for _, r := range records { res := r.Resource() rl, ok := out[res.Equivalent()] From 2f035632f2ea0cdea986b2eb0fe0342913d21209 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Fri, 17 May 2024 13:56:28 -0700 Subject: [PATCH 2/4] Pool scope log maps --- .../otlplog/otlploghttp/internal/transform/log.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go index 79af903542f..0255cbc76e2 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go @@ -67,6 +67,11 @@ func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceL // ScopeLogs returns a slice of OTLP ScopeLogs generated from recoreds. func ScopeLogs(records []log.Record) []*lpb.ScopeLogs { scopeMap := scopeLogsMap(records) + defer func() { + clear(scopeMap) + scopeLogsMapPool.Put(scopeMap) + }() + out := make([]*lpb.ScopeLogs, 0, len(scopeMap)) for _, sl := range scopeMap { out = append(out, sl) @@ -74,8 +79,14 @@ func ScopeLogs(records []log.Record) []*lpb.ScopeLogs { return out } +var scopeLogsMapPool = sync.Pool{ + New: func() any { + return make(map[instrumentation.Scope]*lpb.ScopeLogs) + }, +} + func scopeLogsMap(records []log.Record) map[instrumentation.Scope]*lpb.ScopeLogs { - out := make(map[instrumentation.Scope]*lpb.ScopeLogs) + out := scopeLogsMapPool.Get().(map[instrumentation.Scope]*lpb.ScopeLogs) for _, r := range records { scope := r.InstrumentationScope() sl, ok := out[scope] From 069f53134b2d592a44be6e22e0c561e6d8cc2b29 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Fri, 17 May 2024 14:01:04 -0700 Subject: [PATCH 3/4] Add BenchmarkResourceLogs --- .../otlploghttp/internal/transform/log_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go index 25613601b1c..d345b979940 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go @@ -230,3 +230,14 @@ func TestSeverityNumber(t *testing.T) { assert.Equal(t, want, SeverityNumber(api.Severity(i))) } } + +func BenchmarkResourceLogs(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var out []*lpb.ResourceLogs + for pb.Next() { + out = ResourceLogs(records) + } + _ = out + }) +} From 607a8e97fe01eaba4662282bdf8bb5b9bd3fba23 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Fri, 17 May 2024 14:09:41 -0700 Subject: [PATCH 4/4] Move pool interactions to one location --- .../otlploghttp/internal/transform/log.go | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go index 0255cbc76e2..b1d33a80cce 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go @@ -25,11 +25,12 @@ func ResourceLogs(records []log.Record) []*lpb.ResourceLogs { return nil } - resMap := resourceLogsMap(records) + resMap := resourceLogsMapPool.Get().(map[attribute.Distinct]*lpb.ResourceLogs) defer func() { clear(resMap) resourceLogsMapPool.Put(resMap) }() + resourceLogsMap(&resMap, records) out := make([]*lpb.ResourceLogs, 0, len(resMap)) for _, rl := range resMap { @@ -44,11 +45,10 @@ var resourceLogsMapPool = sync.Pool{ }, } -func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceLogs { - out := resourceLogsMapPool.Get().(map[attribute.Distinct]*lpb.ResourceLogs) +func resourceLogsMap(dst *map[attribute.Distinct]*lpb.ResourceLogs, records []log.Record) { for _, r := range records { res := r.Resource() - rl, ok := out[res.Equivalent()] + rl, ok := (*dst)[res.Equivalent()] if !ok { rl = new(lpb.ResourceLogs) if res.Len() > 0 { @@ -57,20 +57,20 @@ func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceL } } rl.SchemaUrl = res.SchemaURL() - out[res.Equivalent()] = rl + (*dst)[res.Equivalent()] = rl } rl.ScopeLogs = ScopeLogs(records) } - return out } // ScopeLogs returns a slice of OTLP ScopeLogs generated from recoreds. func ScopeLogs(records []log.Record) []*lpb.ScopeLogs { - scopeMap := scopeLogsMap(records) + scopeMap := scopeLogsMapPool.Get().(map[instrumentation.Scope]*lpb.ScopeLogs) defer func() { clear(scopeMap) scopeLogsMapPool.Put(scopeMap) }() + scopeLogsMap(&scopeMap, records) out := make([]*lpb.ScopeLogs, 0, len(scopeMap)) for _, sl := range scopeMap { @@ -85,11 +85,10 @@ var scopeLogsMapPool = sync.Pool{ }, } -func scopeLogsMap(records []log.Record) map[instrumentation.Scope]*lpb.ScopeLogs { - out := scopeLogsMapPool.Get().(map[instrumentation.Scope]*lpb.ScopeLogs) +func scopeLogsMap(dst *map[instrumentation.Scope]*lpb.ScopeLogs, records []log.Record) { for _, r := range records { scope := r.InstrumentationScope() - sl, ok := out[scope] + sl, ok := (*dst)[scope] if !ok { sl = new(lpb.ScopeLogs) var emptyScope instrumentation.Scope @@ -100,11 +99,10 @@ func scopeLogsMap(records []log.Record) map[instrumentation.Scope]*lpb.ScopeLogs } sl.SchemaUrl = scope.SchemaURL } - out[scope] = sl + (*dst)[scope] = sl } sl.LogRecords = append(sl.LogRecords, LogRecord(r)) } - return out } // LogRecord returns an OTLP LogRecord generated from record.