From 4eecb365451529d405ba2caa6234693ad4bdd265 Mon Sep 17 00:00:00 2001 From: Alex Boten <223565+codeboten@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:35:11 -0700 Subject: [PATCH] config: add support for parsing env variables in configuration This replaces the last bit of functionality that was opened in https://github.com/open-telemetry/opentelemetry-go-contrib/pull/4826 to support env variable replacement. Pulled the envprovider.go code from https://github.com/open-telemetry/opentelemetry-collector/blob/main/confmap/provider/envprovider/provider.go Signed-off-by: Alex Boten <223565+codeboten@users.noreply.github.com> --- config/config.go | 15 ++ config/config_test.go | 34 +++ config/envprovider.go | 38 +++ config/testdata/v0.2-env-var.yaml | 428 ++++++++++++++++++++++++++++++ 4 files changed, 515 insertions(+) create mode 100644 config/envprovider.go create mode 100644 config/testdata/v0.2-env-var.yaml diff --git a/config/config.go b/config/config.go index 7031f1cabd5..cc4efb0753b 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ package config // import "go.opentelemetry.io/contrib/config" import ( "context" "errors" + "regexp" "gopkg.in/yaml.v3" @@ -139,6 +140,20 @@ func WithOpenTelemetryConfiguration(cfg OpenTelemetryConfiguration) Configuratio // ParseYAML parses a YAML configuration file into an OpenTelemetryConfiguration. func ParseYAML(file []byte) (*OpenTelemetryConfiguration, error) { + re := regexp.MustCompile(`\$\{([a-zA-Z_][a-zA-Z0-9_]*[-]?.*)\}`) + + replaceEnvVars := func(input []byte) []byte { + return re.ReplaceAllFunc(input, func(s []byte) []byte { + match := re.FindSubmatch(s) + if len(match) < 2 { + return s + } + return replaceEnvVar(string(match[1])) + }) + } + + file = replaceEnvVars(file) + var cfg OpenTelemetryConfiguration err := yaml.Unmarshal(file, &cfg) if err != nil { diff --git a/config/config_test.go b/config/config_test.go index cdc3ddcd45b..903744d269d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -438,6 +438,40 @@ func TestParseYAML(t *testing.T) { } } +func TestParseYAMLWithEnvironmentVariables(t *testing.T) { + tests := []struct { + name string + input string + wantErr error + wantType interface{} + }{ + { + name: "valid v0.2 config with env vars", + input: "v0.2-env-var.yaml", + wantType: &v02OpenTelemetryConfig, + }, + } + + t.Setenv("OTEL_SDK_DISABLED", "false") + t.Setenv("OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT", "4096") + t.Setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := os.ReadFile(filepath.Join("testdata", tt.input)) + require.NoError(t, err) + + got, err := ParseYAML(b) + if tt.wantErr != nil { + require.Equal(t, tt.wantErr.Error(), err.Error()) + } else { + require.NoError(t, err) + assert.Equal(t, tt.wantType, got) + } + }) + } +} + func TestSerializeJSON(t *testing.T) { tests := []struct { name string diff --git a/config/envprovider.go b/config/envprovider.go new file mode 100644 index 00000000000..5821aa49088 --- /dev/null +++ b/config/envprovider.go @@ -0,0 +1,38 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package config // import "go.opentelemetry.io/contrib/config" + +import ( + "os" + "regexp" + "strings" +) + +const ValidationPattern = `^[a-zA-Z_][a-zA-Z0-9_]*$` + +var validationRegexp = regexp.MustCompile(ValidationPattern) + +func replaceEnvVar(uri string) []byte { + envVarName, defaultValuePtr := parseEnvVarURI(uri) + if !validationRegexp.MatchString(envVarName) { + return nil + } + + val, exists := os.LookupEnv(envVarName) + if !exists { + if defaultValuePtr != nil { + val = *defaultValuePtr + } + } + return []byte(val) +} + +func parseEnvVarURI(uri string) (string, *string) { + const defaultSuffix = ":-" + if strings.Contains(uri, defaultSuffix) { + parts := strings.SplitN(uri, defaultSuffix, 2) + return parts[0], &parts[1] + } + return uri, nil +} diff --git a/config/testdata/v0.2-env-var.yaml b/config/testdata/v0.2-env-var.yaml new file mode 100644 index 00000000000..72034c4128c --- /dev/null +++ b/config/testdata/v0.2-env-var.yaml @@ -0,0 +1,428 @@ +# kitchen-sink.yaml demonstrates all configurable surface area, including explanatory comments. +# +# It DOES NOT represent expected real world configuration, as it makes strange configuration +# choices in an effort to exercise the full surface area. +# +# Configuration values are set to their defaults when default values are defined. + +# The file format version +file_format: ${fileformat:-0.2} + +# Configure if the SDK is disabled or not. This is not required to be provided +# to ensure the SDK isn't disabled, the default value when this is not provided +# is for the SDK to be enabled. +# +# Environment variable: OTEL_SDK_DISABLED +disabled: ${OTEL_SDK_DISABLED} + +# Configure general attribute limits. See also tracer_provider.limits, logger_provider.limits. +attribute_limits: + # Configure max attribute value size. + # + # Environment variable: OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT + attribute_value_length_limit: ${OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT} + # Configure max attribute count. + # + # Environment variable: OTEL_ATTRIBUTE_COUNT_LIMIT + attribute_count_limit: 128 + +# Configure logger provider. +logger_provider: + # Configure log record processors. + processors: + # Configure a batch log record processor. + - batch: + # Configure delay interval (in milliseconds) between two consecutive exports. + # + # Environment variable: OTEL_BLRP_SCHEDULE_DELAY + schedule_delay: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # + # Environment variable: OTEL_BLRP_EXPORT_TIMEOUT + export_timeout: 30000 + # Configure maximum queue size. + # + # Environment variable: OTEL_BLRP_MAX_QUEUE_SIZE + max_queue_size: 2048 + # Configure maximum batch size. + # + # Environment variable: OTEL_BLRP_MAX_EXPORT_BATCH_SIZE + max_export_batch_size: 512 + # Configure exporter. + # + # Environment variable: OTEL_LOGS_EXPORTER + exporter: + # Configure exporter to be OTLP. + otlp: + # Configure protocol. + # + # Environment variable: OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + protocol: ${OTEL_EXPORTER_OTLP_PROTOCOL} + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_LOGS_ENDPOINT + endpoint: http://localhost:4318 + # Configure certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE + certificate: /app/cert.pem + # Configure mTLS private client key. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_KEY, OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY + client_key: /app/cert.pem + # Configure mTLS client certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE + client_certificate: /app/cert.pem + # Configure headers. + # + # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_LOGS_HEADERS + headers: + api-key: "1234" + # Configure compression. + # + # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_LOGS_COMPRESSION + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT + timeout: 10000 + # Configure client transport security for the exporter's connection. + # + # Environment variable: OTEL_EXPORTER_OTLP_INSECURE, OTEL_EXPORTER_OTLP_LOGS_INSECURE + insecure: false + # Configure a simple span processor. + - simple: + # Configure exporter. + exporter: + # Configure exporter to be console. + console: {} + # Configure log record limits. See also attribute_limits. + limits: + # Configure max log record attribute value size. Overrides attribute_limits.attribute_value_length_limit. + # + # Environment variable: OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT + attribute_value_length_limit: 4096 + # Configure max log record attribute count. Overrides attribute_limits.attribute_count_limit. + # + # Environment variable: OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT + attribute_count_limit: 128 + +# Configure meter provider. +meter_provider: + # Configure metric readers. + readers: + # Configure a pull-based metric reader. + - pull: + # Configure exporter. + # + # Environment variable: OTEL_METRICS_EXPORTER + exporter: + # Configure exporter to be prometheus. + prometheus: + # Configure host. + # + # Environment variable: OTEL_EXPORTER_PROMETHEUS_HOST + host: localhost + # Configure port. + # + # Environment variable: OTEL_EXPORTER_PROMETHEUS_PORT + port: 9464 + # Configure Prometheus Exporter to produce metrics without a unit suffix or UNIT metadata. + without_units: false + # Configure Prometheus Exporter to produce metrics without a type suffix. + without_type_suffix: false + # Configure Prometheus Exporter to produce metrics without a scope info metric. + without_scope_info: false + # Configure Prometheus Exporter to add resource attributes as metrics attributes. + with_resource_constant_labels: + # Configure resource attributes to be included, in this example attributes starting with service. + included: + - "service*" + # Configure resource attributes to be excluded, in this example attribute service.attr1. + excluded: + - "service.attr1" + # Configure a periodic metric reader. + - periodic: + # Configure delay interval (in milliseconds) between start of two consecutive exports. + # + # Environment variable: OTEL_METRIC_EXPORT_INTERVAL + interval: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # + # Environment variable: OTEL_METRIC_EXPORT_TIMEOUT + timeout: 30000 + # Configure exporter. + # + # Environment variable: OTEL_METRICS_EXPORTER + exporter: + # Configure exporter to be OTLP. + otlp: + # Configure protocol. + # + # Environment variable: OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_METRICS_PROTOCOL + protocol: http/protobuf + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + endpoint: http://localhost:4318 + # Configure certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE + certificate: /app/cert.pem + # Configure mTLS private client key. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_KEY, OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY + client_key: /app/cert.pem + # Configure mTLS client certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE + client_certificate: /app/cert.pem + # Configure headers. + # + # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_METRICS_HEADERS + headers: + api-key: !!str 1234 + # Configure compression. + # + # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_METRICS_COMPRESSION + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_METRICS_TIMEOUT + timeout: 10000 + # Configure client transport security for the exporter's connection. + # + # Environment variable: OTEL_EXPORTER_OTLP_INSECURE, OTEL_EXPORTER_OTLP_METRICS_INSECURE + insecure: false + # Configure temporality preference. + # + # Environment variable: OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE + temporality_preference: delta + # Configure default histogram aggregation. + # + # Environment variable: OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION + default_histogram_aggregation: base2_exponential_bucket_histogram + # Configure a periodic metric reader. + - periodic: + # Configure exporter. + exporter: + # Configure exporter to be console. + console: {} + # Configure views. Each view has a selector which determines the instrument(s) it applies to, and a configuration for the resulting stream(s). + views: + # Configure a view. + - selector: + # Configure instrument name selection criteria. + instrument_name: my-instrument + # Configure instrument type selection criteria. + instrument_type: histogram + # Configure the instrument unit selection criteria. + unit: ms + # Configure meter name selection criteria. + meter_name: my-meter + # Configure meter version selection criteria. + meter_version: 1.0.0 + # Configure meter schema url selection criteria. + meter_schema_url: https://opentelemetry.io/schemas/1.16.0 + # Configure stream. + stream: + # Configure metric name of the resulting stream(s). + name: new_instrument_name + # Configure metric description of the resulting stream(s). + description: new_description + # Configure aggregation of the resulting stream(s). Known values include: default, drop, explicit_bucket_histogram, base2_exponential_bucket_histogram, last_value, sum. + aggregation: + # Configure aggregation to be explicit_bucket_histogram. + explicit_bucket_histogram: + # Configure bucket boundaries. + boundaries: [ 0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 750.0, 1000.0, 2500.0, 5000.0, 7500.0, 10000.0 ] + # Configure record min and max. + record_min_max: true + # Configure attribute keys retained in the resulting stream(s). + attribute_keys: + - key1 + - key2 + +# Configure text map context propagators. +# +# Environment variable: OTEL_PROPAGATORS +propagator: + composite: [tracecontext, baggage, b3, b3multi, jaeger, xray, ottrace] + +# Configure tracer provider. +tracer_provider: + # Configure span processors. + processors: + # Configure a batch span processor. + - batch: + # Configure delay interval (in milliseconds) between two consecutive exports. + # + # Environment variable: OTEL_BSP_SCHEDULE_DELAY + schedule_delay: 5000 + # Configure maximum allowed time (in milliseconds) to export data. + # + # Environment variable: OTEL_BSP_EXPORT_TIMEOUT + export_timeout: 30000 + # Configure maximum queue size. + # + # Environment variable: OTEL_BSP_MAX_QUEUE_SIZE + max_queue_size: 2048 + # Configure maximum batch size. + # + # Environment variable: OTEL_BSP_MAX_EXPORT_BATCH_SIZE + max_export_batch_size: 512 + # Configure exporter. + # + # Environment variable: OTEL_TRACES_EXPORTER + exporter: + # Configure exporter to be OTLP. + otlp: + # Configure protocol. + # + # Environment variable: OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + protocol: http/protobuf + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + endpoint: http://localhost:4318 + # Configure certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE + certificate: /app/cert.pem + # Configure mTLS private client key. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_KEY, OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY + client_key: /app/cert.pem + # Configure mTLS client certificate. + # + # Environment variable: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE + client_certificate: /app/cert.pem + # Configure headers. + # + # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_TRACES_HEADERS + headers: + api-key: !!str 1234 + # Configure compression. + # + # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_TRACES_COMPRESSION + compression: gzip + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_TRACES_TIMEOUT + timeout: 10000 + # Configure client transport security for the exporter's connection. + # + # Environment variable: OTEL_EXPORTER_OTLP_INSECURE, OTEL_EXPORTER_OTLP_TRACES_INSECURE + insecure: false + # Configure a batch span processor. + - batch: + # Configure exporter. + # + # Environment variable: OTEL_TRACES_EXPORTER + exporter: + # Configure exporter to be zipkin. + zipkin: + # Configure endpoint. + # + # Environment variable: OTEL_EXPORTER_ZIPKIN_ENDPOINT + endpoint: http://localhost:9411/api/v2/spans + # Configure max time (in milliseconds) to wait for each export. + # + # Environment variable: OTEL_EXPORTER_ZIPKIN_TIMEOUT + timeout: 10000 + # Configure a simple span processor. + - simple: + # Configure exporter. + exporter: + # Configure exporter to be console. + console: {} + # Configure span limits. See also attribute_limits. + limits: + # Configure max span attribute value size. Overrides attribute_limits.attribute_value_length_limit. + # + # Environment variable: OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT + attribute_value_length_limit: 4096 + # Configure max span attribute count. Overrides attribute_limits.attribute_count_limit. + # + # Environment variable: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + attribute_count_limit: 128 + # Configure max span event count. + # + # Environment variable: OTEL_SPAN_EVENT_COUNT_LIMIT + event_count_limit: 128 + # Configure max span link count. + # + # Environment variable: OTEL_SPAN_LINK_COUNT_LIMIT + link_count_limit: 128 + # Configure max attributes per span event. + # + # Environment variable: OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT + event_attribute_count_limit: 128 + # Configure max attributes per span link. + # + # Environment variable: OTEL_LINK_ATTRIBUTE_COUNT_LIMIT + link_attribute_count_limit: 128 + # Configure the sampler. + sampler: + # Configure sampler to be parent_based. Known values include: always_off, always_on, jaeger_remote, parent_based, trace_id_ratio_based. + # + # Environment variable: OTEL_TRACES_SAMPLER=parentbased_* + parent_based: + # Configure root sampler. + # + # Environment variable: OTEL_TRACES_SAMPLER=parentbased_traceidratio + root: + # Configure sampler to be trace_id_ratio_based. + trace_id_ratio_based: + # Configure trace_id_ratio. + # + # Environment variable: OTEL_TRACES_SAMPLER_ARG=traceidratio=0.0001 + ratio: 0.0001 + # Configure remote_parent_sampled sampler. + remote_parent_sampled: + # Configure sampler to be always_on. + always_on: {} + # Configure remote_parent_not_sampled sampler. + remote_parent_not_sampled: + # Configure sampler to be always_off. + always_off: {} + # Configure local_parent_sampled sampler. + local_parent_sampled: + # Configure sampler to be always_on. + always_on: {} + # Configure local_parent_not_sampled sampler. + local_parent_not_sampled: + # Configure sampler to be always_off. + always_off: {} + +# Configure resource for all signals. +resource: + # Configure resource attributes. + # + # Environment variable: OTEL_RESOURCE_ATTRIBUTES + attributes: + # Configure `service.name` resource attribute + # + # Environment variable: OTEL_SERVICE_NAME + service.name: !!str "unknown_service" + # Configure resource detectors. + detectors: + # Configure attributes provided by resource detectors. + attributes: + # Configure list of attribute key patterns to include from resource detectors. If not set, all attributes are included. + # + # Attribute keys from resource detectors are evaluated to match as follows: + # * If the value of the attribute key exactly matches. + # * If the value of the attribute key matches the wildcard pattern, where '?' matches any single character and '*' matches any number of characters including none. + included: + - process.* + # Configure list of attribute key patterns to exclude from resource detectors. Applies after .resource.detectors.attributes.included (i.e. excluded has higher priority than included). + # + # Attribute keys from resource detectors are evaluated to match as follows: + # * If the value of the attribute key exactly matches. + # * If the value of the attribute key matches the wildcard pattern, where '?' matches any single character and '*' matches any number of characters including none. + excluded: + - process.command_args + # Configure the resource schema URL. + schema_url: https://opentelemetry.io/schemas/1.16.0