From 647ee7e4613d4cc5d38ca538847aef6cfbb703f0 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 15:27:26 -0700 Subject: [PATCH 1/8] Add otlploghttp transform pkg --- exporters/otlp/otlplog/otlploghttp/client.go | 4 +- .../otlp/otlplog/otlploghttp/exporter.go | 8 +- .../otlp/otlplog/otlploghttp/exporter_test.go | 21 +- .../otlploghttp/internal/transform/log.go | 382 +++++++++++++++++- .../internal/transform/log_test.go | 167 ++++++++ 5 files changed, 556 insertions(+), 26 deletions(-) create mode 100644 exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go diff --git a/exporters/otlp/otlplog/otlploghttp/client.go b/exporters/otlp/otlplog/otlploghttp/client.go index 7362e080d93..16747887640 100644 --- a/exporters/otlp/otlplog/otlploghttp/client.go +++ b/exporters/otlp/otlplog/otlploghttp/client.go @@ -10,10 +10,10 @@ import ( ) type client struct { - uploadLogs func(context.Context, *logpb.ResourceLogs) error + uploadLogs func(context.Context, []*logpb.ResourceLogs) error } -func (c *client) UploadLogs(ctx context.Context, rl *logpb.ResourceLogs) error { +func (c *client) UploadLogs(ctx context.Context, rl []*logpb.ResourceLogs) error { if c.uploadLogs != nil { return c.uploadLogs(ctx, rl) } diff --git a/exporters/otlp/otlplog/otlploghttp/exporter.go b/exporters/otlp/otlplog/otlploghttp/exporter.go index ac4c403a970..3f3fd6ef9e9 100644 --- a/exporters/otlp/otlplog/otlploghttp/exporter.go +++ b/exporters/otlp/otlplog/otlploghttp/exporter.go @@ -5,7 +5,6 @@ package otlploghttp // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/o import ( "context" - "errors" "sync/atomic" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/transform" @@ -47,12 +46,11 @@ func (e *Exporter) Export(ctx context.Context, records []log.Record) error { if e.stopped.Load() { return nil } - otlp, err := transformResourceLogs(records) + otlp := transformResourceLogs(records) if otlp != nil { - // Best effort upload of transformable logs. - err = errors.Join(err, e.client.Load().UploadLogs(ctx, otlp)) + return e.client.Load().UploadLogs(ctx, otlp) } - return err + return nil } // Shutdown shuts down the Exporter. Calls to Export or ForceFlush will perform diff --git a/exporters/otlp/otlplog/otlploghttp/exporter_test.go b/exporters/otlp/otlplog/otlploghttp/exporter_test.go index 8779a1b2b9f..418a3148df1 100644 --- a/exporters/otlp/otlplog/otlploghttp/exporter_test.go +++ b/exporters/otlp/otlplog/otlploghttp/exporter_test.go @@ -19,35 +19,24 @@ import ( ) func TestExporterExportErrors(t *testing.T) { - var ( - errUpload = errors.New("upload") - errTForm = errors.New("transform") - ) - + errUpload := errors.New("upload") c := &client{ - uploadLogs: func(context.Context, *logpb.ResourceLogs) error { + uploadLogs: func(context.Context, []*logpb.ResourceLogs) error { return errUpload }, } - orig := transformResourceLogs - transformResourceLogs = func(r []log.Record) (*logpb.ResourceLogs, error) { - return new(logpb.ResourceLogs), errTForm - } - t.Cleanup(func() { transformResourceLogs = orig }) - e, err := newExporter(c, config{}) require.NoError(t, err, "New") err = e.Export(context.Background(), make([]log.Record, 1)) assert.ErrorIs(t, err, errUpload) - assert.ErrorIs(t, err, errTForm) } func TestExporterExport(t *testing.T) { var uploads int c := &client{ - uploadLogs: func(context.Context, *logpb.ResourceLogs) error { + uploadLogs: func(context.Context, []*logpb.ResourceLogs) error { uploads++ return nil }, @@ -55,9 +44,9 @@ func TestExporterExport(t *testing.T) { orig := transformResourceLogs var got []log.Record - transformResourceLogs = func(r []log.Record) (*logpb.ResourceLogs, error) { + transformResourceLogs = func(r []log.Record) []*logpb.ResourceLogs { got = r - return new(logpb.ResourceLogs), nil + return make([]*logpb.ResourceLogs, 1) } t.Cleanup(func() { transformResourceLogs = orig }) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go index bea72f2991a..284c95cb55a 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go @@ -6,12 +6,388 @@ package transform // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/transform" import ( + "time" + + cpb "go.opentelemetry.io/proto/otlp/common/v1" lpb "go.opentelemetry.io/proto/otlp/logs/v1" + rpb "go.opentelemetry.io/proto/otlp/resource/v1" + "go.opentelemetry.io/otel/attribute" + api "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/sdk/instrumentation" "go.opentelemetry.io/otel/sdk/log" ) -func ResourceLogs([]log.Record) (*lpb.ResourceLogs, error) { - // TODO: implement - return nil, nil +// ResourceLogs returns an OTLP ResourceLogs generated from records. If records +// contains invalid ScopeLogs, an error will be returned along with an OTLP +// ResourceLogs that contains partial OTLP ScopeLogs. +func ResourceLogs(records []log.Record) []*lpb.ResourceLogs { + if len(records) == 0 { + return nil + } + + resMap := resourceLogsMap(records) + out := make([]*lpb.ResourceLogs, 0, len(resMap)) + for _, rl := range resMap { + out = append(out, rl) + } + return out +} + +func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceLogs { + out := make(map[attribute.Distinct]*lpb.ResourceLogs) + for _, r := range records { + res := r.Resource() + rl, ok := out[res.Equivalent()] + if !ok { + rl = new(lpb.ResourceLogs) + if res.Len() > 0 { + rl.Resource = &rpb.Resource{ + Attributes: AttrIter(res.Iter()), + } + } + rl.SchemaUrl = res.SchemaURL() + out[res.Equivalent()] = rl + } + rl.ScopeLogs = ScopeLogs(records) + } + return out +} + +// ScopeLogs returns a slice of OTLP ScopeLogs generated from recoreds. If +// records contains invalid log values, an error will be returned along with a +// slice that contains partial OTLP ScopeLogs. +func ScopeLogs(records []log.Record) []*lpb.ScopeLogs { + scopeMap := scopeLogsMap(records) + out := make([]*lpb.ScopeLogs, 0, len(scopeMap)) + for _, sl := range scopeMap { + out = append(out, sl) + } + return out +} + +func scopeLogsMap(records []log.Record) map[instrumentation.Scope]*lpb.ScopeLogs { + out := make(map[instrumentation.Scope]*lpb.ScopeLogs) + for _, r := range records { + scope := r.InstrumentationScope() + sl, ok := out[scope] + if !ok { + sl = new(lpb.ScopeLogs) + var emptyScope instrumentation.Scope + if scope != emptyScope { + sl.Scope = &cpb.InstrumentationScope{ + Name: scope.Name, + Version: scope.Version, + } + sl.SchemaUrl = scope.SchemaURL + } + out[scope] = sl + } + sl.LogRecords = append(sl.LogRecords, LogRecord(r)) + } + return out +} + +// LogRecord returns an OTLP LogRecord generated from record. +func LogRecord(record log.Record) *lpb.LogRecord { + r := &lpb.LogRecord{ + TimeUnixNano: timeUnixNano(record.Timestamp()), + ObservedTimeUnixNano: timeUnixNano(record.ObservedTimestamp()), + SeverityNumber: SeverityNumber(record.Severity()), + SeverityText: record.SeverityText(), + Body: LogAttrValue(record.Body()), + Attributes: make([]*cpb.KeyValue, 0, record.AttributesLen()), + Flags: uint32(record.TraceFlags()), + // TODO: DroppedAttributesCount: /* ... */, + } + record.WalkAttributes(func(kv api.KeyValue) bool { + r.Attributes = append(r.Attributes, LogAttr(kv)) + return true + }) + if tID := record.TraceID(); tID.IsValid() { + r.TraceId = tID[:] + } + if sID := record.SpanID(); sID.IsValid() { + r.SpanId = sID[:] + } + return r +} + +// timeUnixNano returns t as a Unix time, the number of nanoseconds elapsed +// since January 1, 1970 UTC as uint64. +// The result is undefined if the Unix time +// in nanoseconds cannot be represented by an int64 +// (a date before the year 1678 or after 2262). +// timeUnixNano on the zero Time returns 0. +// The result does not depend on the location associated with t. +func timeUnixNano(t time.Time) uint64 { + if t.IsZero() { + return 0 + } + return uint64(t.UnixNano()) +} + +// AttrIter transforms an [attribute.Iterator] into OTLP key-values. +func AttrIter(iter attribute.Iterator) []*cpb.KeyValue { + l := iter.Len() + if l == 0 { + return nil + } + + out := make([]*cpb.KeyValue, 0, l) + for iter.Next() { + out = append(out, Attr(iter.Attribute())) + } + return out +} + +// Attrs transforms a slice of [attribute.KeyValue] into OTLP key-values. +func Attrs(attrs []attribute.KeyValue) []*cpb.KeyValue { + if len(attrs) == 0 { + return nil + } + + out := make([]*cpb.KeyValue, 0, len(attrs)) + for _, kv := range attrs { + out = append(out, Attr(kv)) + } + return out +} + +// Attr transforms an [attribute.KeyValue] into an OTLP key-value. +func Attr(kv attribute.KeyValue) *cpb.KeyValue { + return &cpb.KeyValue{Key: string(kv.Key), Value: AttrValue(kv.Value)} +} + +// AttrValue transforms an [attribute.Value] into an OTLP AnyValue. +func AttrValue(v attribute.Value) *cpb.AnyValue { + av := new(cpb.AnyValue) + switch v.Type() { + case attribute.BOOL: + av.Value = &cpb.AnyValue_BoolValue{ + BoolValue: v.AsBool(), + } + case attribute.BOOLSLICE: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: boolSliceValues(v.AsBoolSlice()), + }, + } + case attribute.INT64: + av.Value = &cpb.AnyValue_IntValue{ + IntValue: v.AsInt64(), + } + case attribute.INT64SLICE: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: int64SliceValues(v.AsInt64Slice()), + }, + } + case attribute.FLOAT64: + av.Value = &cpb.AnyValue_DoubleValue{ + DoubleValue: v.AsFloat64(), + } + case attribute.FLOAT64SLICE: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: float64SliceValues(v.AsFloat64Slice()), + }, + } + case attribute.STRING: + av.Value = &cpb.AnyValue_StringValue{ + StringValue: v.AsString(), + } + case attribute.STRINGSLICE: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: stringSliceValues(v.AsStringSlice()), + }, + } + default: + av.Value = &cpb.AnyValue_StringValue{ + StringValue: "INVALID", + } + } + return av +} + +func boolSliceValues(vals []bool) []*cpb.AnyValue { + converted := make([]*cpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = &cpb.AnyValue{ + Value: &cpb.AnyValue_BoolValue{ + BoolValue: v, + }, + } + } + return converted +} + +func int64SliceValues(vals []int64) []*cpb.AnyValue { + converted := make([]*cpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = &cpb.AnyValue{ + Value: &cpb.AnyValue_IntValue{ + IntValue: v, + }, + } + } + return converted +} + +func float64SliceValues(vals []float64) []*cpb.AnyValue { + converted := make([]*cpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = &cpb.AnyValue{ + Value: &cpb.AnyValue_DoubleValue{ + DoubleValue: v, + }, + } + } + return converted +} + +func stringSliceValues(vals []string) []*cpb.AnyValue { + converted := make([]*cpb.AnyValue, len(vals)) + for i, v := range vals { + converted[i] = &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{ + StringValue: v, + }, + } + } + return converted +} + +// Attrs transforms a slice of [api.KeyValue] into OTLP key-values. +func LogAttrs(attrs []api.KeyValue) []*cpb.KeyValue { + if len(attrs) == 0 { + return nil + } + + out := make([]*cpb.KeyValue, 0, len(attrs)) + for _, kv := range attrs { + out = append(out, LogAttr(kv)) + } + return out +} + +// LogAttr transforms an [api.KeyValue] into an OTLP key-value. +func LogAttr(attr api.KeyValue) *cpb.KeyValue { + return &cpb.KeyValue{ + Key: attr.Key, + Value: LogAttrValue(attr.Value), + } +} + +// LogAttrValues transforms a slice of [api.Value] into an OTLP []AnyValue. +func LogAttrValues(vals []api.Value) []*cpb.AnyValue { + if len(vals) == 0 { + return nil + } + + out := make([]*cpb.AnyValue, 0, len(vals)) + for _, v := range vals { + out = append(out, LogAttrValue(v)) + } + return out +} + +// LogAttrValue transforms an [api.Value] into an OTLP AnyValue. +func LogAttrValue(v api.Value) *cpb.AnyValue { + av := new(cpb.AnyValue) + switch v.Kind() { + case api.KindBool: + av.Value = &cpb.AnyValue_BoolValue{ + BoolValue: v.AsBool(), + } + case api.KindInt64: + av.Value = &cpb.AnyValue_IntValue{ + IntValue: v.AsInt64(), + } + case api.KindFloat64: + av.Value = &cpb.AnyValue_DoubleValue{ + DoubleValue: v.AsFloat64(), + } + case api.KindString: + av.Value = &cpb.AnyValue_StringValue{ + StringValue: v.AsString(), + } + case api.KindBytes: + av.Value = &cpb.AnyValue_BytesValue{ + BytesValue: v.AsBytes(), + } + case api.KindSlice: + av.Value = &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: LogAttrValues(v.AsSlice()), + }, + } + case api.KindMap: + av.Value = &cpb.AnyValue_KvlistValue{ + KvlistValue: &cpb.KeyValueList{ + Values: LogAttrs(v.AsMap()), + }, + } + default: + av.Value = &cpb.AnyValue_StringValue{ + StringValue: "INVALID", + } + } + return av +} + +// SeverityNumber transforms a [log.Severity] into an OTLP SeverityNumber. +func SeverityNumber(s api.Severity) lpb.SeverityNumber { + switch s { + case api.SeverityTrace: + return lpb.SeverityNumber_SEVERITY_NUMBER_TRACE + case api.SeverityTrace2: + return lpb.SeverityNumber_SEVERITY_NUMBER_TRACE2 + case api.SeverityTrace3: + return lpb.SeverityNumber_SEVERITY_NUMBER_TRACE3 + case api.SeverityTrace4: + return lpb.SeverityNumber_SEVERITY_NUMBER_TRACE4 + case api.SeverityDebug: + return lpb.SeverityNumber_SEVERITY_NUMBER_DEBUG + case api.SeverityDebug2: + return lpb.SeverityNumber_SEVERITY_NUMBER_DEBUG2 + case api.SeverityDebug3: + return lpb.SeverityNumber_SEVERITY_NUMBER_DEBUG3 + case api.SeverityDebug4: + return lpb.SeverityNumber_SEVERITY_NUMBER_DEBUG4 + case api.SeverityInfo: + return lpb.SeverityNumber_SEVERITY_NUMBER_INFO + case api.SeverityInfo2: + return lpb.SeverityNumber_SEVERITY_NUMBER_INFO2 + case api.SeverityInfo3: + return lpb.SeverityNumber_SEVERITY_NUMBER_INFO3 + case api.SeverityInfo4: + return lpb.SeverityNumber_SEVERITY_NUMBER_INFO4 + case api.SeverityWarn: + return lpb.SeverityNumber_SEVERITY_NUMBER_WARN + case api.SeverityWarn2: + return lpb.SeverityNumber_SEVERITY_NUMBER_WARN2 + case api.SeverityWarn3: + return lpb.SeverityNumber_SEVERITY_NUMBER_WARN3 + case api.SeverityWarn4: + return lpb.SeverityNumber_SEVERITY_NUMBER_WARN4 + case api.SeverityError: + return lpb.SeverityNumber_SEVERITY_NUMBER_ERROR + case api.SeverityError2: + return lpb.SeverityNumber_SEVERITY_NUMBER_ERROR2 + case api.SeverityError3: + return lpb.SeverityNumber_SEVERITY_NUMBER_ERROR3 + case api.SeverityError4: + return lpb.SeverityNumber_SEVERITY_NUMBER_ERROR4 + case api.SeverityFatal: + return lpb.SeverityNumber_SEVERITY_NUMBER_FATAL + case api.SeverityFatal2: + return lpb.SeverityNumber_SEVERITY_NUMBER_FATAL2 + case api.SeverityFatal3: + return lpb.SeverityNumber_SEVERITY_NUMBER_FATAL3 + case api.SeverityFatal4: + return lpb.SeverityNumber_SEVERITY_NUMBER_FATAL4 + } + return lpb.SeverityNumber_SEVERITY_NUMBER_UNSPECIFIED } diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go new file mode 100644 index 00000000000..6a417a002de --- /dev/null +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go @@ -0,0 +1,167 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package transform + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + cpb "go.opentelemetry.io/proto/otlp/common/v1" + lpb "go.opentelemetry.io/proto/otlp/logs/v1" + + api "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/trace" +) + +var ( + // Sat Jan 01 2000 00:00:00 GMT+0000. + ts = time.Date(2000, time.January, 0o1, 0, 0, 0, 0, time.FixedZone("GMT", 0)) + obs = ts.Add(30 * time.Second) + + alice = api.String("user", "alice") + bob = api.String("user", "bob") + + pbAlice = &cpb.KeyValue{Key: "user", Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "alice"}, + }} + pbBob = &cpb.KeyValue{Key: "user", Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "bob"}, + }} + + sevA = api.SeverityInfo + sevB = api.SeverityError + + pbSevA = lpb.SeverityNumber_SEVERITY_NUMBER_INFO + pbSevB = lpb.SeverityNumber_SEVERITY_NUMBER_ERROR + + bodyA = api.StringValue("a") + bodyB = api.StringValue("b") + + pbBodyA = &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{ + StringValue: "a", + }, + } + pbBodyB = &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{ + StringValue: "b", + }, + } + + spanIDA = []byte{0, 0, 0, 0, 0, 0, 0, 1} + spanIDB = []byte{0, 0, 0, 0, 0, 0, 0, 2} + traceIDA = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + traceIDB = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2} + flagsA = byte(1) + flagsB = byte(0) + + records = func() []log.Record { + r0 := new(log.Record) + r0.SetTimestamp(ts) + r0.SetObservedTimestamp(obs) + r0.SetSeverity(sevA) + r0.SetSeverityText("A") + r0.SetBody(bodyA) + r0.SetAttributes(alice) + r0.SetTraceID(trace.TraceID(traceIDA)) + r0.SetSpanID(trace.SpanID(spanIDA)) + r0.SetTraceFlags(trace.TraceFlags(flagsA)) + + r1 := new(log.Record) + r1.SetTimestamp(ts) + r1.SetObservedTimestamp(obs) + r1.SetSeverity(sevA) + r1.SetSeverityText("A") + r1.SetBody(bodyA) + r1.SetAttributes(bob) + r1.SetTraceID(trace.TraceID(traceIDA)) + r1.SetSpanID(trace.SpanID(spanIDA)) + r1.SetTraceFlags(trace.TraceFlags(flagsA)) + + r2 := new(log.Record) + r2.SetTimestamp(ts) + r2.SetObservedTimestamp(obs) + r2.SetSeverity(sevB) + r2.SetSeverityText("B") + r2.SetBody(bodyB) + r2.SetAttributes(alice) + r2.SetTraceID(trace.TraceID(traceIDB)) + r2.SetSpanID(trace.SpanID(spanIDB)) + r2.SetTraceFlags(trace.TraceFlags(flagsB)) + + r3 := new(log.Record) + r3.SetTimestamp(ts) + r3.SetObservedTimestamp(obs) + r3.SetSeverity(sevB) + r3.SetSeverityText("B") + r3.SetBody(bodyB) + r3.SetAttributes(bob) + r3.SetTraceID(trace.TraceID(traceIDB)) + r3.SetSpanID(trace.SpanID(spanIDB)) + r3.SetTraceFlags(trace.TraceFlags(flagsB)) + + return []log.Record{*r0, *r1, *r2, *r3} + }() + + pbLogRecords = []*lpb.LogRecord{ + { + TimeUnixNano: uint64(ts.UnixNano()), + ObservedTimeUnixNano: uint64(obs.UnixNano()), + SeverityNumber: pbSevA, + SeverityText: "A", + Body: pbBodyA, + Attributes: []*cpb.KeyValue{pbAlice}, + Flags: uint32(flagsA), + TraceId: traceIDA, + SpanId: spanIDA, + }, + { + TimeUnixNano: uint64(ts.UnixNano()), + ObservedTimeUnixNano: uint64(obs.UnixNano()), + SeverityNumber: pbSevA, + SeverityText: "A", + Body: pbBodyA, + Attributes: []*cpb.KeyValue{pbBob}, + Flags: uint32(flagsA), + TraceId: traceIDA, + SpanId: spanIDA, + }, + { + TimeUnixNano: uint64(ts.UnixNano()), + ObservedTimeUnixNano: uint64(obs.UnixNano()), + SeverityNumber: pbSevB, + SeverityText: "B", + Body: pbBodyB, + Attributes: []*cpb.KeyValue{pbAlice}, + Flags: uint32(flagsB), + TraceId: traceIDB, + SpanId: spanIDB, + }, + { + TimeUnixNano: uint64(ts.UnixNano()), + ObservedTimeUnixNano: uint64(obs.UnixNano()), + SeverityNumber: pbSevB, + SeverityText: "B", + Body: pbBodyB, + Attributes: []*cpb.KeyValue{pbBob}, + Flags: uint32(flagsB), + TraceId: traceIDB, + SpanId: spanIDB, + }, + } + + pbScopeLogs = &lpb.ScopeLogs{LogRecords: pbLogRecords} + + pbResourceLogs = &lpb.ResourceLogs{ + ScopeLogs: []*lpb.ScopeLogs{pbScopeLogs}, + } +) + +func TestTransformations(t *testing.T) { + want := []*lpb.ResourceLogs{pbResourceLogs} + assert.Equal(t, want, ResourceLogs(records)) +} From e6fc4b296e3688523066d9b0558a2ba59aa94c49 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 16:15:14 -0700 Subject: [PATCH 2/8] Rename TestTransformations to TestResourceLogs --- .../otlp/otlplog/otlploghttp/internal/transform/log_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go index 6a417a002de..e7de2172114 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go @@ -161,7 +161,7 @@ var ( } ) -func TestTransformations(t *testing.T) { +func TestResourceLogs(t *testing.T) { want := []*lpb.ResourceLogs{pbResourceLogs} assert.Equal(t, want, ResourceLogs(records)) } From 40e6c2d5a5d7bab38b3f6a5e663148daade829f9 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 16:15:24 -0700 Subject: [PATCH 3/8] Add TestAttrTransforms --- .../internal/transform/attr_test.go | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 exporters/otlp/otlplog/otlploghttp/internal/transform/attr_test.go diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/attr_test.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/attr_test.go new file mode 100644 index 00000000000..99320b4eae0 --- /dev/null +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/attr_test.go @@ -0,0 +1,183 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/otel/attribute" + cpb "go.opentelemetry.io/proto/otlp/common/v1" +) + +var ( + attrBool = attribute.Bool("bool", true) + attrBoolSlice = attribute.BoolSlice("bool slice", []bool{true, false}) + attrInt = attribute.Int("int", 1) + attrIntSlice = attribute.IntSlice("int slice", []int{-1, 1}) + attrInt64 = attribute.Int64("int64", 1) + attrInt64Slice = attribute.Int64Slice("int64 slice", []int64{-1, 1}) + attrFloat64 = attribute.Float64("float64", 1) + attrFloat64Slice = attribute.Float64Slice("float64 slice", []float64{-1, 1}) + attrString = attribute.String("string", "o") + attrStringSlice = attribute.StringSlice("string slice", []string{"o", "n"}) + attrInvalid = attribute.KeyValue{ + Key: attribute.Key("invalid"), + Value: attribute.Value{}, + } + + valBoolTrue = &cpb.AnyValue{Value: &cpb.AnyValue_BoolValue{BoolValue: true}} + valBoolFalse = &cpb.AnyValue{Value: &cpb.AnyValue_BoolValue{BoolValue: false}} + valBoolSlice = &cpb.AnyValue{Value: &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: []*cpb.AnyValue{valBoolTrue, valBoolFalse}, + }, + }} + valIntOne = &cpb.AnyValue{Value: &cpb.AnyValue_IntValue{IntValue: 1}} + valIntNOne = &cpb.AnyValue{Value: &cpb.AnyValue_IntValue{IntValue: -1}} + valIntSlice = &cpb.AnyValue{Value: &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: []*cpb.AnyValue{valIntNOne, valIntOne}, + }, + }} + valDblOne = &cpb.AnyValue{Value: &cpb.AnyValue_DoubleValue{DoubleValue: 1}} + valDblNOne = &cpb.AnyValue{Value: &cpb.AnyValue_DoubleValue{DoubleValue: -1}} + valDblSlice = &cpb.AnyValue{Value: &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: []*cpb.AnyValue{valDblNOne, valDblOne}, + }, + }} + valStrO = &cpb.AnyValue{Value: &cpb.AnyValue_StringValue{StringValue: "o"}} + valStrN = &cpb.AnyValue{Value: &cpb.AnyValue_StringValue{StringValue: "n"}} + valStrSlice = &cpb.AnyValue{Value: &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: []*cpb.AnyValue{valStrO, valStrN}, + }, + }} + + kvBool = &cpb.KeyValue{Key: "bool", Value: valBoolTrue} + kvBoolSlice = &cpb.KeyValue{Key: "bool slice", Value: valBoolSlice} + kvInt = &cpb.KeyValue{Key: "int", Value: valIntOne} + kvIntSlice = &cpb.KeyValue{Key: "int slice", Value: valIntSlice} + kvInt64 = &cpb.KeyValue{Key: "int64", Value: valIntOne} + kvInt64Slice = &cpb.KeyValue{Key: "int64 slice", Value: valIntSlice} + kvFloat64 = &cpb.KeyValue{Key: "float64", Value: valDblOne} + kvFloat64Slice = &cpb.KeyValue{Key: "float64 slice", Value: valDblSlice} + kvString = &cpb.KeyValue{Key: "string", Value: valStrO} + kvStringSlice = &cpb.KeyValue{Key: "string slice", Value: valStrSlice} + kvInvalid = &cpb.KeyValue{ + Key: "invalid", + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "INVALID"}, + }, + } +) + +func TestAttrTransforms(t *testing.T) { + type attrTest struct { + name string + in []attribute.KeyValue + want []*cpb.KeyValue + } + + for _, test := range []attrTest{ + {"nil", nil, nil}, + {"empty", []attribute.KeyValue{}, nil}, + { + "invalid", + []attribute.KeyValue{attrInvalid}, + []*cpb.KeyValue{kvInvalid}, + }, + { + "bool", + []attribute.KeyValue{attrBool}, + []*cpb.KeyValue{kvBool}, + }, + { + "bool slice", + []attribute.KeyValue{attrBoolSlice}, + []*cpb.KeyValue{kvBoolSlice}, + }, + { + "int", + []attribute.KeyValue{attrInt}, + []*cpb.KeyValue{kvInt}, + }, + { + "int slice", + []attribute.KeyValue{attrIntSlice}, + []*cpb.KeyValue{kvIntSlice}, + }, + { + "int64", + []attribute.KeyValue{attrInt64}, + []*cpb.KeyValue{kvInt64}, + }, + { + "int64 slice", + []attribute.KeyValue{attrInt64Slice}, + []*cpb.KeyValue{kvInt64Slice}, + }, + { + "float64", + []attribute.KeyValue{attrFloat64}, + []*cpb.KeyValue{kvFloat64}, + }, + { + "float64 slice", + []attribute.KeyValue{attrFloat64Slice}, + []*cpb.KeyValue{kvFloat64Slice}, + }, + { + "string", + []attribute.KeyValue{attrString}, + []*cpb.KeyValue{kvString}, + }, + { + "string slice", + []attribute.KeyValue{attrStringSlice}, + []*cpb.KeyValue{kvStringSlice}, + }, + { + "all", + []attribute.KeyValue{ + attrBool, + attrBoolSlice, + attrInt, + attrIntSlice, + attrInt64, + attrInt64Slice, + attrFloat64, + attrFloat64Slice, + attrString, + attrStringSlice, + attrInvalid, + }, + []*cpb.KeyValue{ + kvBool, + kvBoolSlice, + kvInt, + kvIntSlice, + kvInt64, + kvInt64Slice, + kvFloat64, + kvFloat64Slice, + kvString, + kvStringSlice, + kvInvalid, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Run("Attrs", func(t *testing.T) { + assert.ElementsMatch(t, test.want, Attrs(test.in)) + }) + t.Run("AttrIter", func(t *testing.T) { + s := attribute.NewSet(test.in...) + assert.ElementsMatch(t, test.want, AttrIter(s.Iter())) + }) + }) + } +} From efbdf096b36b15a99a434953428ebc92d4ccf82f Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 16:34:18 -0700 Subject: [PATCH 4/8] Add TestLogAttrs --- .../internal/transform/log_attr_test.go | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 exporters/otlp/otlplog/otlploghttp/internal/transform/log_attr_test.go diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_attr_test.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_attr_test.go new file mode 100644 index 00000000000..570a6927cad --- /dev/null +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_attr_test.go @@ -0,0 +1,146 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/otel/log" + cpb "go.opentelemetry.io/proto/otlp/common/v1" +) + +var ( + logAttrBool = log.Bool("bool", true) + logAttrInt = log.Int("int", 1) + logAttrInt64 = log.Int64("int64", 1) + logAttrFloat64 = log.Float64("float64", 1) + logAttrString = log.String("string", "o") + logAttrBytes = log.Bytes("bytes", []byte("test")) + logAttrSlice = log.Slice("slice", log.BoolValue(true)) + logAttrMap = log.Map("map", logAttrString) + logAttrEmpty = log.Empty("") + + kvBytes = &cpb.KeyValue{ + Key: "bytes", + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_BytesValue{ + BytesValue: []byte("test"), + }, + }, + } + kvSlice = &cpb.KeyValue{ + Key: "slice", + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_ArrayValue{ + ArrayValue: &cpb.ArrayValue{ + Values: []*cpb.AnyValue{valBoolTrue}, + }, + }, + }, + } + kvMap = &cpb.KeyValue{ + Key: "map", + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_KvlistValue{ + KvlistValue: &cpb.KeyValueList{ + Values: []*cpb.KeyValue{kvString}, + }, + }, + }, + } + kvEmpty = &cpb.KeyValue{ + Value: &cpb.AnyValue{ + Value: &cpb.AnyValue_StringValue{StringValue: "INVALID"}, + }, + } +) + +func TestLogAttrs(t *testing.T) { + type logAttrTest struct { + name string + in []log.KeyValue + want []*cpb.KeyValue + } + + for _, test := range []logAttrTest{ + {"nil", nil, nil}, + {"len(0)", []log.KeyValue{}, nil}, + { + "empty", + []log.KeyValue{logAttrEmpty}, + []*cpb.KeyValue{kvEmpty}, + }, + { + "bool", + []log.KeyValue{logAttrBool}, + []*cpb.KeyValue{kvBool}, + }, + { + "int", + []log.KeyValue{logAttrInt}, + []*cpb.KeyValue{kvInt}, + }, + { + "int64", + []log.KeyValue{logAttrInt64}, + []*cpb.KeyValue{kvInt64}, + }, + { + "float64", + []log.KeyValue{logAttrFloat64}, + []*cpb.KeyValue{kvFloat64}, + }, + { + "string", + []log.KeyValue{logAttrString}, + []*cpb.KeyValue{kvString}, + }, + { + "bytes", + []log.KeyValue{logAttrBytes}, + []*cpb.KeyValue{kvBytes}, + }, + { + "slice", + []log.KeyValue{logAttrSlice}, + []*cpb.KeyValue{kvSlice}, + }, + { + "map", + []log.KeyValue{logAttrMap}, + []*cpb.KeyValue{kvMap}, + }, + { + "all", + []log.KeyValue{ + logAttrBool, + logAttrInt, + logAttrInt64, + logAttrFloat64, + logAttrString, + logAttrBytes, + logAttrSlice, + logAttrMap, + logAttrEmpty, + }, + []*cpb.KeyValue{ + kvBool, + kvInt, + kvInt64, + kvFloat64, + kvString, + kvBytes, + kvSlice, + kvMap, + kvEmpty, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + assert.ElementsMatch(t, test.want, LogAttrs(test.in)) + }) + } +} From 7b61bf0107086dd9cf6be711c54c7d50a4b65b95 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 16:40:43 -0700 Subject: [PATCH 5/8] Add TestSeverityNumber --- .../otlplog/otlploghttp/internal/transform/log_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go index e7de2172114..2ce4557f7e9 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go @@ -165,3 +165,11 @@ func TestResourceLogs(t *testing.T) { want := []*lpb.ResourceLogs{pbResourceLogs} assert.Equal(t, want, ResourceLogs(records)) } + +func TestSeverityNumber(t *testing.T) { + for i := 0; i <= int(api.SeverityFatal4); i++ { + want := lpb.SeverityNumber(i) + want += lpb.SeverityNumber_SEVERITY_NUMBER_UNSPECIFIED + assert.Equal(t, want, SeverityNumber(api.Severity(i))) + } +} From 49c6d45bd36b71f5c7af6d8c6f65bfdfa3b302e0 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 16:45:25 -0700 Subject: [PATCH 6/8] Update go mod --- exporters/otlp/otlplog/otlploghttp/go.mod | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exporters/otlp/otlplog/otlploghttp/go.mod b/exporters/otlp/otlplog/otlploghttp/go.mod index 8299e87a262..4b5cfba48c6 100644 --- a/exporters/otlp/otlplog/otlploghttp/go.mod +++ b/exporters/otlp/otlplog/otlploghttp/go.mod @@ -5,7 +5,11 @@ go 1.21 require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/otel v1.25.0 + go.opentelemetry.io/otel/log v0.0.1-alpha + go.opentelemetry.io/otel/sdk v1.24.0 go.opentelemetry.io/otel/sdk/log v0.0.0-20240403115316-6c6e1e7416e9 + go.opentelemetry.io/otel/trace v1.25.0 go.opentelemetry.io/proto/otlp v1.1.0 ) @@ -14,11 +18,7 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go.opentelemetry.io/otel v1.25.0 // indirect - go.opentelemetry.io/otel/log v0.0.1-alpha // indirect go.opentelemetry.io/otel/metric v1.25.0 // indirect - go.opentelemetry.io/otel/sdk v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.25.0 // indirect golang.org/x/sys v0.19.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect From b4864456c1f6ca39463397fe48b8c4d94105ceec Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 10 Apr 2024 16:49:58 -0700 Subject: [PATCH 7/8] Fix comments --- .../otlploghttp/internal/transform/log.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go index 284c95cb55a..a59536aa704 100644 --- a/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go +++ b/exporters/otlp/otlplog/otlploghttp/internal/transform/log.go @@ -18,9 +18,7 @@ import ( "go.opentelemetry.io/otel/sdk/log" ) -// ResourceLogs returns an OTLP ResourceLogs generated from records. If records -// contains invalid ScopeLogs, an error will be returned along with an OTLP -// ResourceLogs that contains partial OTLP ScopeLogs. +// ResourceLogs returns an slice of OTLP ResourceLogs generated from records. func ResourceLogs(records []log.Record) []*lpb.ResourceLogs { if len(records) == 0 { return nil @@ -54,9 +52,7 @@ func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceL return out } -// ScopeLogs returns a slice of OTLP ScopeLogs generated from recoreds. If -// records contains invalid log values, an error will be returned along with a -// slice that contains partial OTLP ScopeLogs. +// ScopeLogs returns a slice of OTLP ScopeLogs generated from recoreds. func ScopeLogs(records []log.Record) []*lpb.ScopeLogs { scopeMap := scopeLogsMap(records) out := make([]*lpb.ScopeLogs, 0, len(scopeMap)) @@ -114,12 +110,10 @@ func LogRecord(record log.Record) *lpb.LogRecord { } // timeUnixNano returns t as a Unix time, the number of nanoseconds elapsed -// since January 1, 1970 UTC as uint64. -// The result is undefined if the Unix time -// in nanoseconds cannot be represented by an int64 -// (a date before the year 1678 or after 2262). -// timeUnixNano on the zero Time returns 0. -// The result does not depend on the location associated with t. +// since January 1, 1970 UTC as uint64. The result is undefined if the Unix +// time in nanoseconds cannot be represented by an int64 (a date before the +// year 1678 or after 2262). timeUnixNano on the zero Time returns 0. The +// result does not depend on the location associated with t. func timeUnixNano(t time.Time) uint64 { if t.IsZero() { return 0 From 2ef33c8fa4d68197f4885c51aee6a6421202f70d Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Thu, 11 Apr 2024 07:33:16 -0700 Subject: [PATCH 8/8] Apply feedback --- exporters/otlp/otlplog/otlploghttp/exporter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporters/otlp/otlplog/otlploghttp/exporter.go b/exporters/otlp/otlplog/otlploghttp/exporter.go index 3f3fd6ef9e9..e852b766501 100644 --- a/exporters/otlp/otlplog/otlploghttp/exporter.go +++ b/exporters/otlp/otlplog/otlploghttp/exporter.go @@ -47,10 +47,10 @@ func (e *Exporter) Export(ctx context.Context, records []log.Record) error { return nil } otlp := transformResourceLogs(records) - if otlp != nil { - return e.client.Load().UploadLogs(ctx, otlp) + if otlp == nil { + return nil } - return nil + return e.client.Load().UploadLogs(ctx, otlp) } // Shutdown shuts down the Exporter. Calls to Export or ForceFlush will perform