otlpmetricgrpc: Keep metadata for gRPC in context (#5892)

When using otlpmetricgrpc, we can inject metadata dynamically with
`metadata.AppendToOutgoingContext`. However when specifying headers with
`WithHeaders`, it becomes impossible to add additional metadata.

For example, the code below sends additional metadata in the header
```go
exp := otlpmetricgrpc.New(context.Background())

ctx := context.Background()
ctx = metadata.AppendToOutgoingContext(ctx, "dynamic-key", "value")
exp.Export(ctx, &metricdata.ResourceMetrics{}))
```


But when using `WithHeader` like below, no additional metadata will be
sent
```go
exp := otlpmetricgrpc.New(context.Background(), otlpmetricgrpc.WithHeaders("custom-key", "value"))

ctx := context.Background()
ctx = metadata.AppendToOutgoingContext(ctx, "dynamic-key", "value")
exp.Export(ctx, &metricdata.ResourceMetrics{}))
```

To eliminate this inconsistency, keep the metadata in the context and
send them in the header.

---------

Co-authored-by: Tyler Yahn <[email protected]>

CHANGELOG.md

@@ -22,6 +22,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 - Global MeterProvider registration unwraps global instrument Observers, the undocumented Unwrap() methods are now private. (#5881)
 
+### Changed
+
+- `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc` now keeps the metadata already present in the context when `WithHeaders` is used. (#5892)
+
 <!-- Released section -->
 <!-- Don't change this section unless doing release -->
 

exporters/otlp/otlpmetric/otlpmetricgrpc/client.go

@@ -155,7 +155,12 @@ func (c *client) exportContext(parent context.Context) (context.Context, context
 	}
 
 	if c.metadata.Len() > 0 {
-		ctx = metadata.NewOutgoingContext(ctx, c.metadata)
+		md := c.metadata
+		if outMD, ok := metadata.FromOutgoingContext(ctx); ok {
+			md = metadata.Join(md, outMD)
+		}
+
+		ctx = metadata.NewOutgoingContext(ctx, md)
 	}
 
 	return ctx, cancel

exporters/otlp/otlpmetric/otlpmetricgrpc/client_test.go

@@ -13,6 +13,7 @@ import (
 	"google.golang.org/genproto/googleapis/rpc/errdetails"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/durationpb"
 
@@ -207,14 +208,18 @@ func TestConfig(t *testing.T) {
 		headers := map[string]string{key: "custom-value"}
 		exp, coll := factoryFunc(nil, WithHeaders(headers))
 		t.Cleanup(coll.Shutdown)
+
 		ctx := context.Background()
+		additionalKey := "additional-custom-header"
+		ctx = metadata.AppendToOutgoingContext(ctx, additionalKey, "additional-value")
 		require.NoError(t, exp.Export(ctx, &metricdata.ResourceMetrics{}))
 		// Ensure everything is flushed.
 		require.NoError(t, exp.Shutdown(ctx))
 
 		got := coll.Headers()
 		require.Regexp(t, "OTel Go OTLP over gRPC metrics exporter/[01]\\..*", got)
 		require.Contains(t, got, key)
+		require.Contains(t, got, additionalKey)
 		assert.Equal(t, []string{headers[key]}, got[key])
 	})