diff --git a/src/cli.rs b/src/cli.rs index 38648c7ad..1205fdfb2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -529,7 +529,9 @@ impl FromArgMatches for Cli { self.kafka_host = m.get_one::(Self::KAFKA_HOST).cloned(); self.kafka_group = m.get_one::(Self::KAFKA_GROUP).cloned(); self.kafka_client_id = m.get_one::(Self::KAFKA_CLIENT_ID).cloned(); - self.kafka_security_protocol = m.get_one::(Self::KAFKA_SECURITY_PROTOCOL).cloned(); + self.kafka_security_protocol = m + .get_one::(Self::KAFKA_SECURITY_PROTOCOL) + .cloned(); self.kafka_partitions = m.get_one::(Self::KAFKA_PARTITIONS).cloned(); self.tls_cert_path = m.get_one::(Self::TLS_CERT).cloned(); diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs index 2c70dca45..7cc0bfb66 100644 --- a/src/handlers/http/ingest.rs +++ b/src/handlers/http/ingest.rs @@ -20,6 +20,7 @@ use super::logstream::error::{CreateStreamError, StreamError}; use super::modal::utils::ingest_utils::{flatten_and_push_logs, push_logs}; use super::users::dashboards::DashboardError; use super::users::filters::FiltersError; +use super::{otel_logs, otel_metrics, otel_traces}; use crate::event::{ self, error::EventError, @@ -105,7 +106,7 @@ pub async fn ingest_internal_stream(stream_name: String, body: Bytes) -> Result< // Handler for POST /v1/logs to ingest OTEL logs // ingests events by extracting stream name from header // creates if stream does not exist -pub async fn handle_otel_ingestion( +pub async fn handle_otel_logs_ingestion( req: HttpRequest, body: Bytes, ) -> Result { @@ -114,9 +115,69 @@ pub async fn handle_otel_ingestion( .iter() .find(|&(key, _)| key == STREAM_NAME_HEADER_KEY) { - let stream_name = stream_name.to_str().unwrap(); - create_stream_if_not_exists(stream_name, &StreamType::UserDefined.to_string()).await?; - push_logs(stream_name, &req, &body).await?; + let stream_name = stream_name.to_str().unwrap().to_owned(); + create_stream_if_not_exists(&stream_name, &StreamType::UserDefined.to_string()).await?; + + //custom flattening required for otel logs + let mut json = otel_logs::flatten_otel_logs(&body); + for record in json.iter_mut() { + let body: Bytes = serde_json::to_vec(record).unwrap().into(); + push_logs(&stream_name, &req, &body).await?; + } + } else { + return Err(PostError::Header(ParseHeaderError::MissingStreamName)); + } + Ok(HttpResponse::Ok().finish()) +} + +// Handler for POST /v1/metrics to ingest OTEL metrics +// ingests events by extracting stream name from header +// creates if stream does not exist +pub async fn handle_otel_metrics_ingestion( + req: HttpRequest, + body: Bytes, +) -> Result { + if let Some((_, stream_name)) = req + .headers() + .iter() + .find(|&(key, _)| key == STREAM_NAME_HEADER_KEY) + { + let stream_name = stream_name.to_str().unwrap().to_owned(); + create_stream_if_not_exists(&stream_name, &StreamType::UserDefined.to_string()).await?; + + //custom flattening required for otel metrics + let mut json = otel_metrics::flatten_otel_metrics(&body); + for record in json.iter_mut() { + let body: Bytes = serde_json::to_vec(record).unwrap().into(); + push_logs(&stream_name, &req, &body).await?; + } + } else { + return Err(PostError::Header(ParseHeaderError::MissingStreamName)); + } + Ok(HttpResponse::Ok().finish()) +} + +// Handler for POST /v1/traces to ingest OTEL traces +// ingests events by extracting stream name from header +// creates if stream does not exist +pub async fn handle_otel_traces_ingestion( + req: HttpRequest, + body: Bytes, +) -> Result { + if let Some((_, stream_name)) = req + .headers() + .iter() + .find(|&(key, _)| key == STREAM_NAME_HEADER_KEY) + { + let stream_name = stream_name.to_str().unwrap().to_owned(); + create_stream_if_not_exists(&stream_name, &StreamType::UserDefined.to_string()).await?; + + //custom flattening required for otel traces + let mut json = otel_traces::flatten_otel_traces(&body); + for record in json.iter_mut() { + let body: Bytes = serde_json::to_vec(record).unwrap().into(); + push_logs(&stream_name, &req, &body).await?; + } } else { return Err(PostError::Header(ParseHeaderError::MissingStreamName)); } diff --git a/src/handlers/http/mod.rs b/src/handlers/http/mod.rs index f627b613a..e7ae83212 100644 --- a/src/handlers/http/mod.rs +++ b/src/handlers/http/mod.rs @@ -37,6 +37,10 @@ pub mod logstream; pub mod middleware; pub mod modal; pub mod oidc; +pub mod otel; +pub mod otel_logs; +pub mod otel_metrics; +pub mod otel_traces; pub mod query; pub mod rbac; pub mod role; diff --git a/src/handlers/http/modal/server.rs b/src/handlers/http/modal/server.rs index 6c0ec9fd8..1ebc1b75f 100644 --- a/src/handlers/http/modal/server.rs +++ b/src/handlers/http/modal/server.rs @@ -398,7 +398,7 @@ impl Server { web::resource("/logs") .route( web::post() - .to(ingest::handle_otel_ingestion) + .to(ingest::handle_otel_logs_ingestion) .authorize_for_stream(Action::Ingest), ) .app_data(web::PayloadConfig::default().limit(MAX_EVENT_PAYLOAD_SIZE)), @@ -407,7 +407,7 @@ impl Server { web::resource("/metrics") .route( web::post() - .to(ingest::handle_otel_ingestion) + .to(ingest::handle_otel_metrics_ingestion) .authorize_for_stream(Action::Ingest), ) .app_data(web::PayloadConfig::default().limit(MAX_EVENT_PAYLOAD_SIZE)), @@ -416,7 +416,7 @@ impl Server { web::resource("/traces") .route( web::post() - .to(ingest::handle_otel_ingestion) + .to(ingest::handle_otel_traces_ingestion) .authorize_for_stream(Action::Ingest), ) .app_data(web::PayloadConfig::default().limit(MAX_EVENT_PAYLOAD_SIZE)), diff --git a/src/handlers/http/modal/utils/ingest_utils.rs b/src/handlers/http/modal/utils/ingest_utils.rs index 82f00cf80..04d268ba1 100644 --- a/src/handlers/http/modal/utils/ingest_utils.rs +++ b/src/handlers/http/modal/utils/ingest_utils.rs @@ -16,7 +16,10 @@ * */ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; use actix_web::HttpRequest; use arrow_schema::Field; @@ -30,8 +33,9 @@ use crate::{ format::{self, EventFormat}, }, handlers::{ - http::{ingest::PostError, kinesis}, - LOG_SOURCE_KEY, LOG_SOURCE_KINESIS, PREFIX_META, PREFIX_TAGS, SEPARATOR, + http::{ingest::PostError, kinesis, otel_logs, otel_metrics, otel_traces}, + LOG_SOURCE_KEY, LOG_SOURCE_KINESIS, LOG_SOURCE_OTEL_LOGS, LOG_SOURCE_OTEL_METRICS, + LOG_SOURCE_OTEL_TRACES, PREFIX_META, PREFIX_TAGS, SEPARATOR, }, metadata::STREAM_INFO, storage::StreamType, @@ -43,14 +47,33 @@ pub async fn flatten_and_push_logs( body: Bytes, stream_name: &str, ) -> Result<(), PostError> { - let log_source = req - .headers() - .get(LOG_SOURCE_KEY) - .map(|header| header.to_str().unwrap_or_default()) - .unwrap_or_default(); - if log_source == LOG_SOURCE_KINESIS { - let json = kinesis::flatten_kinesis_logs(&body); - for record in json.iter() { + //flatten logs + if let Some((_, log_source)) = req.headers().iter().find(|&(key, _)| key == LOG_SOURCE_KEY) { + let mut json: Vec> = Vec::new(); + let log_source: String = log_source.to_str().unwrap().to_owned(); + match log_source.as_str() { + LOG_SOURCE_KINESIS => json = kinesis::flatten_kinesis_logs(&body), + + //custom flattening required for otel logs + LOG_SOURCE_OTEL_LOGS => { + json = otel_logs::flatten_otel_logs(&body); + } + + //custom flattening required for otel metrics + LOG_SOURCE_OTEL_METRICS => { + json = otel_metrics::flatten_otel_metrics(&body); + } + + //custom flattening required for otel traces + LOG_SOURCE_OTEL_TRACES => { + json = otel_traces::flatten_otel_traces(&body); + } + _ => { + tracing::warn!("Unknown log source: {}", log_source); + push_logs(stream_name, &req, &body).await?; + } + } + for record in json.iter_mut() { let body: Bytes = serde_json::to_vec(record).unwrap().into(); push_logs(stream_name, &req, &body).await?; } diff --git a/src/handlers/http/otel.rs b/src/handlers/http/otel.rs new file mode 100644 index 000000000..2b0baae62 --- /dev/null +++ b/src/handlers/http/otel.rs @@ -0,0 +1,148 @@ +/* + * Parseable Server (C) 2022 - 2024 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +pub mod proto; +use proto::common::v1::KeyValue; +use serde_json::Value; +use std::collections::BTreeMap; +// Value can be one of types - String, Bool, Int, Double, ArrayValue, AnyValue, KeyValueList, Byte +pub fn collect_json_from_any_value( + key: &String, + value: super::otel::proto::common::v1::Value, +) -> BTreeMap { + let mut value_json: BTreeMap = BTreeMap::new(); + insert_if_some(&mut value_json, key, &value.str_val); + insert_bool_if_some(&mut value_json, key, &value.bool_val); + insert_if_some(&mut value_json, key, &value.int_val); + insert_number_if_some(&mut value_json, key, &value.double_val); + + //ArrayValue is a vector of AnyValue + //traverse by recursively calling the same function + if value.array_val.is_some() { + let array_val = value.array_val.as_ref().unwrap(); + let values = &array_val.values; + for value in values { + let array_value_json = collect_json_from_any_value(key, value.clone()); + for key in array_value_json.keys() { + value_json.insert( + format!( + "{}_{}", + key.to_owned(), + value_to_string(array_value_json[key].to_owned()) + ), + array_value_json[key].to_owned(), + ); + } + } + } + + //KeyValueList is a vector of KeyValue + //traverse through each element in the vector + if value.kv_list_val.is_some() { + let kv_list_val = value.kv_list_val.unwrap(); + for key_value in kv_list_val.values { + let value = key_value.value; + if value.is_some() { + let value = value.unwrap(); + let key_value_json = collect_json_from_any_value(key, value); + + for key in key_value_json.keys() { + value_json.insert( + format!( + "{}_{}_{}", + key.to_owned(), + key_value.key, + value_to_string(key_value_json[key].to_owned()) + ), + key_value_json[key].to_owned(), + ); + } + } + } + } + insert_if_some(&mut value_json, key, &value.bytes_val); + + value_json +} + +//traverse through Value by calling function ollect_json_from_any_value +pub fn collect_json_from_values( + values: &Option, + key: &String, +) -> BTreeMap { + let mut value_json: BTreeMap = BTreeMap::new(); + + for value in values.iter() { + value_json = collect_json_from_any_value(key, value.clone()); + } + + value_json +} + +pub fn value_to_string(value: serde_json::Value) -> String { + match value.clone() { + e @ Value::Number(_) | e @ Value::Bool(_) => e.to_string(), + Value::String(s) => s, + _ => "".to_string(), + } +} + +pub fn flatten_attributes(attributes: &Vec) -> BTreeMap { + let mut attributes_json: BTreeMap = BTreeMap::new(); + for attribute in attributes { + let key = &attribute.key; + let value = &attribute.value; + let value_json = collect_json_from_values(value, &key.to_string()); + for key in value_json.keys() { + attributes_json.insert(key.to_owned(), value_json[key].to_owned()); + } + } + attributes_json +} + +pub fn insert_if_some( + map: &mut BTreeMap, + key: &str, + option: &Option, +) { + if let Some(value) = option { + map.insert(key.to_string(), Value::String(value.to_string())); + } +} + +pub fn insert_number_if_some(map: &mut BTreeMap, key: &str, option: &Option) { + if let Some(value) = option { + if let Some(number) = serde_json::Number::from_f64(*value) { + map.insert(key.to_string(), Value::Number(number)); + } + } +} + +pub fn insert_bool_if_some(map: &mut BTreeMap, key: &str, option: &Option) { + if let Some(value) = option { + map.insert(key.to_string(), Value::Bool(*value)); + } +} + +pub fn insert_attributes(map: &mut BTreeMap, attributes: &Option>) { + if let Some(attrs) = attributes { + let attributes_json = flatten_attributes(attrs); + for (key, value) in attributes_json { + map.insert(key, value); + } + } +} diff --git a/src/handlers/http/otel/opentelemetry.proto.metrics.v1.rs b/src/handlers/http/otel/opentelemetry.proto.metrics.v1.rs new file mode 100644 index 000000000..016d0a341 --- /dev/null +++ b/src/handlers/http/otel/opentelemetry.proto.metrics.v1.rs @@ -0,0 +1,742 @@ +/* + * Parseable Server (C) 2022 - 2024 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +// This file was generated by protoc-gen-rust-protobuf. The file was edited after the generation. + // All the repeated fields were changed to Option>. + +/// MetricsData represents the metrics data that can be stored in a persistent +/// storage, OR can be embedded by other protocols that transfer OTLP metrics +/// data but do not implement the OTLP protocol. +/// +/// MetricsData +/// └─── ResourceMetrics +/// ├── Resource +/// ├── SchemaURL +/// └── ScopeMetrics +/// ├── Scope +/// ├── SchemaURL +/// └── Metric +/// ├── Name +/// ├── Description +/// ├── Unit +/// └── data +/// ├── Gauge +/// ├── Sum +/// ├── Histogram +/// ├── ExponentialHistogram +/// └── Summary +/// +/// The main difference between this message and collector protocol is that +/// in this message there will not be any "control" or "metadata" specific to +/// OTLP protocol. +/// +/// When new fields are added into this message, the OTLP request MUST be updated +/// as well. + use crate::handlers::http::otel::proto::common::v1::InstrumentationScope; + use crate::handlers::http::otel::proto::common::v1::KeyValue; + use crate::handlers::http::otel::proto::resource::v1::Resource; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug)] + #[serde(rename_all = "camelCase")] +pub struct MetricsData { + /// An array of ResourceMetrics. + /// For data coming from a single resource this array will typically contain + /// one element. Intermediary nodes that receive data from multiple origins + /// typically batch the data before forwarding further and in that case this + /// array will contain multiple elements. + #[serde(rename = "resourceMetrics")] + pub resource_metrics: Option>, +} +/// A collection of ScopeMetrics from a Resource. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ResourceMetrics { + /// The resource for the metrics in this message. + /// If this field is not set then no resource info is known. + pub resource: Option, + /// A list of metrics that originate from a resource. + #[serde(rename = "scopeMetrics")] + pub scope_metrics: Option>, + /// The Schema URL, if known. This is the identifier of the Schema that the resource data + /// is recorded in. Notably, the last part of the URL path is the version number of the + /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see + /// + /// This schema_url applies to the data in the "resource" field. It does not apply + /// to the data in the "scope_metrics" field which have their own schema_url field. + #[serde(rename = "schemaUrl")] + pub schema_url: Option, +} +/// A collection of Metrics produced by an Scope. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ScopeMetrics { + /// The instrumentation scope information for the metrics in this message. + /// Semantically when InstrumentationScope isn't set, it is equivalent with + /// an empty instrumentation scope name (unknown). + pub scope: Option, + /// A list of metrics that originate from an instrumentation library. + #[serde(rename = "metrics")] + pub metrics: Vec, + /// The Schema URL, if known. This is the identifier of the Schema that the metric data + /// is recorded in. Notably, the last part of the URL path is the version number of the + /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see + /// + /// This schema_url applies to all metrics in the "metrics" field. + #[serde(rename = "schemaUrl")] + pub schema_url: Option, +} +/// Defines a Metric which has one or more timeseries. The following is a +/// brief summary of the Metric data model. For more details, see: +/// +/// +/// +/// The data model and relation between entities is shown in the +/// diagram below. Here, "DataPoint" is the term used to refer to any +/// one of the specific data point value types, and "points" is the term used +/// to refer to any one of the lists of points contained in the Metric. +/// +/// - Metric is composed of a metadata and data. +/// - Metadata part contains a name, description, unit. +/// - Data is one of the possible types (Sum, Gauge, Histogram, Summary). +/// - DataPoint contains timestamps, attributes, and one of the possible value type +/// fields. +/// +/// Metric +/// +------------+ +/// |name | +/// |description | +/// |unit | +------------------------------------+ +/// |data |---> |Gauge, Sum, Histogram, Summary, ... | +/// +------------+ +------------------------------------+ +/// +/// Data \[One of Gauge, Sum, Histogram, Summary, ...\] +/// +-----------+ +/// |... | // Metadata about the Data. +/// |points |--+ +/// +-----------+ | +/// | +---------------------------+ +/// | |DataPoint 1 | +/// v |+------+------+ +------+ | +/// +-----+ ||label |label |...|label | | +/// | 1 |-->||value1|value2|...|valueN| | +/// +-----+ |+------+------+ +------+ | +/// | . | |+-----+ | +/// | . | ||value| | +/// | . | |+-----+ | +/// | . | +---------------------------+ +/// | . | . +/// | . | . +/// | . | . +/// | . | +---------------------------+ +/// | . | |DataPoint M | +/// +-----+ |+------+------+ +------+ | +/// | M |-->||label |label |...|label | | +/// +-----+ ||value1|value2|...|valueN| | +/// |+------+------+ +------+ | +/// |+-----+ | +/// ||value| | +/// |+-----+ | +/// +---------------------------+ +/// +/// Each distinct type of DataPoint represents the output of a specific +/// aggregation function, the result of applying the DataPoint's +/// associated function of to one or more measurements. +/// +/// All DataPoint types have three common fields: +/// - Attributes includes key-value pairs associated with the data point +/// - TimeUnixNano is required, set to the end time of the aggregation +/// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints +/// having an AggregationTemporality field, as discussed below. +/// +/// Both TimeUnixNano and StartTimeUnixNano values are expressed as +/// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. +/// +/// # TimeUnixNano +/// +/// This field is required, having consistent interpretation across +/// DataPoint types. TimeUnixNano is the moment corresponding to when +/// the data point's aggregate value was captured. +/// +/// Data points with the 0 value for TimeUnixNano SHOULD be rejected +/// by consumers. +/// +/// # StartTimeUnixNano +/// +/// StartTimeUnixNano in general allows detecting when a sequence of +/// observations is unbroken. This field indicates to consumers the +/// start time for points with cumulative and delta +/// AggregationTemporality, and it should be included whenever possible +/// to support correct rate calculation. Although it may be omitted +/// when the start time is truly unknown, setting StartTimeUnixNano is +/// strongly encouraged. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Metric { + /// name of the metric. + pub name: Option, + /// description of the metric, which can be used in documentation. + + pub description: Option, + /// unit in which the metric value is reported. Follows the format + /// described by + pub unit: Option, + /// Additional metadata attributes that describe the metric. \[Optional\]. + /// Attributes are non-identifying. + /// Consumers SHOULD NOT need to be aware of these attributes. + /// These attributes MAY be used to encode information allowing + /// for lossless roundtrip translation to / from another data model. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub metadata: Option>, + /// Data determines the aggregation type (if any) of the metric, what is the + /// reported value type for the data points, as well as the relatationship to + /// the time interval over which they are reported. + pub gauge: Option, + pub sum: Option, + pub histogram: Option, + pub exponential_histogram: Option, + pub summary: Option, +} +/// Gauge represents the type of a scalar metric that always exports the +/// "current value" for every data point. It should be used for an "unknown" +/// aggregation. +/// +/// A Gauge does not support different aggregation temporalities. Given the +/// aggregation is unknown, points cannot be combined using the same +/// aggregation, regardless of aggregation temporalities. Therefore, +/// AggregationTemporality is not included. Consequently, this also means +/// "StartTimeUnixNano" is ignored for all data points. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Gauge { + pub data_points: Option>, +} +/// Sum represents the type of a scalar metric that is calculated as a sum of all +/// reported measurements over a time interval. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Sum { + pub data_points: Option>, + /// aggregation_temporality describes if the aggregator reports delta changes + /// since last report time, or cumulative changes since a fixed start time. + + pub aggregation_temporality: Option, + /// If "true" means that the sum is monotonic. + + pub is_monotonic: Option, +} +/// Histogram represents the type of a metric that is calculated by aggregating +/// as a Histogram of all reported measurements over a time interval. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Histogram { + + pub data_points: Option>, + /// aggregation_temporality describes if the aggregator reports delta changes + /// since last report time, or cumulative changes since a fixed start time. + + pub aggregation_temporality: Option, +} +/// ExponentialHistogram represents the type of a metric that is calculated by aggregating +/// as a ExponentialHistogram of all reported double measurements over a time interval. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ExponentialHistogram { + pub data_points: Option>, + /// aggregation_temporality describes if the aggregator reports delta changes + /// since last report time, or cumulative changes since a fixed start time. + + pub aggregation_temporality: Option, +} +/// Summary metric data are used to convey quantile summaries, +/// a Prometheus (see: ) +/// and OpenMetrics (see: ) +/// data type. These data points cannot always be merged in a meaningful way. +/// While they can be useful in some applications, histogram data points are +/// recommended for new applications. +/// Summary metrics do not have an aggregation temporality field. This is +/// because the count and sum fields of a SummaryDataPoint are assumed to be +/// cumulative values. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Summary { + pub data_points: Option>, +} +/// NumberDataPoint is a single data point in a timeseries that describes the +/// time-varying scalar value of a metric. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct NumberDataPoint { + /// The set of key/value pairs that uniquely identify the timeseries from + /// where this point belongs. The list may be empty (may contain 0 elements). + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// StartTimeUnixNano is optional but strongly encouraged, see the + /// the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub start_time_unix_nano: Option, + /// TimeUnixNano is required, see the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub time_unix_nano: Option, + /// (Optional) List of exemplars collected from + /// measurements that were used to form the data point + pub exemplars: Option>, + /// Flags that apply to this specific data point. See DataPointFlags + /// for the available flags and their meaning. + pub flags: Option, + /// The value itself. A point is considered invalid when one of the recognized + /// value fields is not present inside this oneof. + pub as_double: Option, + pub as_int: Option, +} +/// HistogramDataPoint is a single data point in a timeseries that describes the +/// time-varying values of a Histogram. A Histogram contains summary statistics +/// for a population of values, it may optionally contain the distribution of +/// those values across a set of buckets. +/// +/// If the histogram contains the distribution of values, then both +/// "explicit_bounds" and "bucket counts" fields must be defined. +/// If the histogram does not contain the distribution of values, then both +/// "explicit_bounds" and "bucket_counts" must be omitted and only "count" and +/// "sum" are known. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct HistogramDataPoint { + /// The set of key/value pairs that uniquely identify the timeseries from + /// where this point belongs. The list may be empty (may contain 0 elements). + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// StartTimeUnixNano is optional but strongly encouraged, see the + /// the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub start_time_unix_nano: Option, + /// TimeUnixNano is required, see the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub time_unix_nano: Option, + /// count is the number of values in the population. Must be non-negative. This + /// value must be equal to the sum of the "count" fields in buckets if a + /// histogram is provided. + pub count: Option, + /// sum of the values in the population. If count is zero then this field + /// must be zero. + /// + /// Note: Sum should only be filled out when measuring non-negative discrete + /// events, and is assumed to be monotonic over the values of these events. + /// Negative events *can* be recorded, but sum should not be filled out when + /// doing so. This is specifically to enforce compatibility w/ OpenMetrics, + /// see: + pub sum: Option, + /// bucket_counts is an optional field contains the count values of histogram + /// for each bucket. + /// + /// The sum of the bucket_counts must equal the value in the count field. + /// + /// The number of elements in bucket_counts array must be by one greater than + /// the number of elements in explicit_bounds array. + pub bucket_counts: Option>, + /// explicit_bounds specifies buckets with explicitly defined bounds for values. + /// + /// The boundaries for bucket at index i are: + /// + /// (-infinity, explicit_bounds\[i]\] for i == 0 + /// (explicit_bounds\[i-1\], explicit_bounds\[i]\] for 0 < i < size(explicit_bounds) + /// (explicit_bounds\[i-1\], +infinity) for i == size(explicit_bounds) + /// + /// The values in the explicit_bounds array must be strictly increasing. + /// + /// Histogram buckets are inclusive of their upper boundary, except the last + /// bucket where the boundary is at infinity. This format is intentionally + /// compatible with the OpenMetrics histogram definition. + pub explicit_bounds: Option>, + /// (Optional) List of exemplars collected from + /// measurements that were used to form the data point + pub exemplars: Option>, + /// Flags that apply to this specific data point. See DataPointFlags + /// for the available flags and their meaning. + pub flags: Option, + /// min is the minimum value over (start_time, end_time]. + pub min: Option, + /// max is the maximum value over (start_time, end_time]. + pub max: Option, +} +/// ExponentialHistogramDataPoint is a single data point in a timeseries that describes the +/// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains +/// summary statistics for a population of values, it may optionally contain the +/// distribution of those values across a set of buckets. +/// +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ExponentialHistogramDataPoint { + /// The set of key/value pairs that uniquely identify the timeseries from + /// where this point belongs. The list may be empty (may contain 0 elements). + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// StartTimeUnixNano is optional but strongly encouraged, see the + /// the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub start_time_unix_nano: Option, + /// TimeUnixNano is required, see the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub time_unix_nano: Option, + /// count is the number of values in the population. Must be + /// non-negative. This value must be equal to the sum of the "bucket_counts" + /// values in the positive and negative Buckets plus the "zero_count" field. + pub count: Option, + /// sum of the values in the population. If count is zero then this field + /// must be zero. + /// + /// Note: Sum should only be filled out when measuring non-negative discrete + /// events, and is assumed to be monotonic over the values of these events. + /// Negative events *can* be recorded, but sum should not be filled out when + /// doing so. This is specifically to enforce compatibility w/ OpenMetrics, + /// see: + pub sum: Option, + /// scale describes the resolution of the histogram. Boundaries are + /// located at powers of the base, where: + /// + /// base = (2^(2^-scale)) + /// + /// The histogram bucket identified by `index`, a signed integer, + /// contains values that are greater than (base^index) and + /// less than or equal to (base^(index+1)). + /// + /// The positive and negative ranges of the histogram are expressed + /// separately. Negative values are mapped by their absolute value + /// into the negative range using the same scale as the positive range. + /// + /// scale is not restricted by the protocol, as the permissible + /// values depend on the range of the data. + pub scale: Option, + /// zero_count is the count of values that are either exactly zero or + /// within the region considered zero by the instrumentation at the + /// tolerated degree of precision. This bucket stores values that + /// cannot be expressed using the standard exponential formula as + /// well as values that have been rounded to zero. + /// + /// Implementations MAY consider the zero bucket to have probability + /// mass equal to (zero_count / count). + pub zero_count: Option, + /// positive carries the positive range of exponential bucket counts. + pub positive: Option, + /// negative carries the negative range of exponential bucket counts. + pub negative: Option, + /// Flags that apply to this specific data point. See DataPointFlags + /// for the available flags and their meaning. + pub flags: Option, + /// (Optional) List of exemplars collected from + /// measurements that were used to form the data point + pub exemplars: Option>, + /// min is the minimum value over (start_time, end_time]. + pub min: Option, + /// max is the maximum value over (start_time, end_time]. + pub max: Option, + /// ZeroThreshold may be optionally set to convey the width of the zero + /// region. Where the zero region is defined as the closed interval + /// \[-ZeroThreshold, ZeroThreshold\]. + /// When ZeroThreshold is 0, zero count bucket stores values that cannot be + /// expressed using the standard exponential formula as well as values that + /// have been rounded to zero. + pub zero_threshold: Option, +} +/// Nested message and enum types in `ExponentialHistogramDataPoint`. +pub mod exponential_histogram_data_point { + use serde::{Deserialize, Serialize}; + /// Buckets are a set of bucket counts, encoded in a contiguous array + /// of counts. + #[derive(Serialize, Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct Buckets { + /// Offset is the bucket index of the first entry in the bucket_counts array. + /// + /// Note: This uses a varint encoding as a simple form of compression. + pub offset: Option, + /// bucket_counts is an array of count values, where bucket_counts\[i\] carries + /// the count of the bucket at index (offset+i). bucket_counts\[i\] is the count + /// of values greater than base^(offset+i) and less than or equal to + /// base^(offset+i+1). + /// + /// Note: By contrast, the explicit HistogramDataPoint uses + /// fixed64. This field is expected to have many buckets, + /// especially zeros, so uint64 has been selected to ensure + /// varint encoding. + pub bucket_counts: Option>, + } +} +/// SummaryDataPoint is a single data point in a timeseries that describes the +/// time-varying values of a Summary metric. The count and sum fields represent +/// cumulative values. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct SummaryDataPoint { + /// The set of key/value pairs that uniquely identify the timeseries from + /// where this point belongs. The list may be empty (may contain 0 elements). + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// StartTimeUnixNano is optional but strongly encouraged, see the + /// the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub start_time_unix_nano: Option, + /// TimeUnixNano is required, see the detailed comments above Metric. + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub time_unix_nano: Option, + /// count is the number of values in the population. Must be non-negative. + pub count: Option, + /// sum of the values in the population. If count is zero then this field + /// must be zero. + /// + /// Note: Sum should only be filled out when measuring non-negative discrete + /// events, and is assumed to be monotonic over the values of these events. + /// Negative events *can* be recorded, but sum should not be filled out when + /// doing so. This is specifically to enforce compatibility w/ OpenMetrics, + /// see: + pub sum: Option, + /// (Optional) list of values at different quantiles of the distribution calculated + /// from the current snapshot. The quantiles must be strictly increasing. + pub quantile_values: Option>, + /// Flags that apply to this specific data point. See DataPointFlags + /// for the available flags and their meaning. + pub flags: Option, +} +/// Nested message and enum types in `SummaryDataPoint`. +pub mod summary_data_point { + use serde::{Deserialize, Deserializer, Serialize}; + /// Represents the value at a given quantile of a distribution. + /// + /// To record Min and Max values following conventions are used: + /// - The 1.0 quantile is equivalent to the maximum value observed. + /// - The 0.0 quantile is equivalent to the minimum value observed. + /// + /// See the following issue for more context: + /// + #[derive(Serialize, Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + pub struct ValueAtQuantile { + /// The quantile of a distribution. Must be in the interval + /// \[0.0, 1.0\]. + pub quantile: Option, + /// The value at the given quantile of a distribution. + /// + /// Quantile values must NOT be negative. + #[serde(deserialize_with = "deserialize_f64_or_nan")] + pub value: Option, + } + + fn deserialize_f64_or_nan<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, + { + struct StringOrFloatVisitor; + impl<'de> serde::de::Visitor<'de> for StringOrFloatVisitor + { + type Value = Option; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result + { + formatter.write_str("a string or a floating-point number") + } + fn visit_str(self, value: &str) -> Result where E: serde::de::Error, + { + if value == "NaN" + { + Ok(Some(f64::NAN)) + } else { + value.parse::().map(Some).map_err(E::custom) + } } + fn visit_f64(self, value: f64) -> Result where E: serde::de::Error, + { + Ok(Some(value)) + } + } + deserializer.deserialize_any(StringOrFloatVisitor) } +} +/// A representation of an exemplar, which is a sample input measurement. +/// Exemplars also hold information about the environment when the measurement +/// was recorded, for example the span and trace ID of the active span when the +/// exemplar was recorded. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Exemplar { + /// The set of key/value pairs that were filtered out by the aggregator, but + /// recorded alongside the original measurement. Only key/value pairs that were + /// filtered out by the aggregator should be included + pub filtered_attributes: Option>, + /// time_unix_nano is the exact time when this exemplar was recorded + /// + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + /// 1970. + pub time_unix_nano: Option, + /// (Optional) Span ID of the exemplar trace. + /// span_id may be missing if the measurement is not recorded inside a trace + /// or if the trace is not sampled. + pub span_id: Option, + /// (Optional) Trace ID of the exemplar trace. + /// trace_id may be missing if the measurement is not recorded inside a trace + /// or if the trace is not sampled. + pub trace_id: Option, + /// The value of the measurement that was recorded. An exemplar is + /// considered invalid when one of the recognized value fields is not present + /// inside this oneof. + pub as_double: Option, + pub as_int: Option, +} +/// AggregationTemporality defines how a metric aggregator reports aggregated +/// values. It describes how those values relate to the time interval over +/// which they are aggregated. +#[repr(i32)] +pub enum AggregationTemporality { + /// UNSPECIFIED is the default AggregationTemporality, it MUST not be used. + Unspecified = 0, + /// DELTA is an AggregationTemporality for a metric aggregator which reports + /// changes since last report time. Successive metrics contain aggregation of + /// values from continuous and non-overlapping intervals. + /// + /// The values for a DELTA metric are based only on the time interval + /// associated with one measurement cycle. There is no dependency on + /// previous measurements like is the case for CUMULATIVE metrics. + /// + /// For example, consider a system measuring the number of requests that + /// it receives and reports the sum of these requests every second as a + /// DELTA metric: + /// + /// 1. The system starts receiving at time=t_0. + /// 2. A request is received, the system measures 1 request. + /// 3. A request is received, the system measures 1 request. + /// 4. A request is received, the system measures 1 request. + /// 5. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0 to + /// t_0+1 with a value of 3. + /// 6. A request is received, the system measures 1 request. + /// 7. A request is received, the system measures 1 request. + /// 8. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0+1 to + /// t_0+2 with a value of 2. + Delta = 1, + /// CUMULATIVE is an AggregationTemporality for a metric aggregator which + /// reports changes since a fixed start time. This means that current values + /// of a CUMULATIVE metric depend on all previous measurements since the + /// start time. Because of this, the sender is required to retain this state + /// in some form. If this state is lost or invalidated, the CUMULATIVE metric + /// values MUST be reset and a new fixed start time following the last + /// reported measurement time sent MUST be used. + /// + /// For example, consider a system measuring the number of requests that + /// it receives and reports the sum of these requests every second as a + /// CUMULATIVE metric: + /// + /// 1. The system starts receiving at time=t_0. + /// 2. A request is received, the system measures 1 request. + /// 3. A request is received, the system measures 1 request. + /// 4. A request is received, the system measures 1 request. + /// 5. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0 to + /// t_0+1 with a value of 3. + /// 6. A request is received, the system measures 1 request. + /// 7. A request is received, the system measures 1 request. + /// 8. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_0 to + /// t_0+2 with a value of 5. + /// 9. The system experiences a fault and loses state. + /// 10. The system recovers and resumes receiving at time=t_1. + /// 11. A request is received, the system measures 1 request. + /// 12. The 1 second collection cycle ends. A metric is exported for the + /// number of requests received over the interval of time t_1 to + /// t_0+1 with a value of 1. + /// + /// Note: Even though, when reporting changes since last report time, using + /// CUMULATIVE is valid, it is not recommended. This may cause problems for + /// systems that do not use start_time to determine when the aggregation + /// value was reset (e.g. Prometheus). + Cumulative = 2, +} +impl AggregationTemporality { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "AGGREGATION_TEMPORALITY_UNSPECIFIED", + Self::Delta => "AGGREGATION_TEMPORALITY_DELTA", + Self::Cumulative => "AGGREGATION_TEMPORALITY_CUMULATIVE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "AGGREGATION_TEMPORALITY_UNSPECIFIED" => Some(Self::Unspecified), + "AGGREGATION_TEMPORALITY_DELTA" => Some(Self::Delta), + "AGGREGATION_TEMPORALITY_CUMULATIVE" => Some(Self::Cumulative), + _ => None, + } + } +} +/// DataPointFlags is defined as a protobuf 'uint32' type and is to be used as a +/// bit-field representing 32 distinct boolean flags. Each flag defined in this +/// enum is a bit-mask. To test the presence of a single flag in the flags of +/// a data point, for example, use an expression like: +/// +/// (point.flags & DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) == DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK +/// +#[repr(i32)] +pub enum DataPointFlags { + /// The zero value for the enum. Should not be used for comparisons. + /// Instead use bitwise "and" with the appropriate mask as shown above. + DoNotUse = 0, + /// This DataPoint is valid but has no recorded value. This value + /// SHOULD be used to reflect explicitly missing data in a series, as + /// for an equivalent to the Prometheus "staleness marker". + NoRecordedValueMask = 1, +} +impl DataPointFlags { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::DoNotUse => "DATA_POINT_FLAGS_DO_NOT_USE", + Self::NoRecordedValueMask => "DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "DATA_POINT_FLAGS_DO_NOT_USE" => Some(Self::DoNotUse), + "DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK" => Some(Self::NoRecordedValueMask), + _ => None, + } + } +} diff --git a/src/handlers/http/otel/opentelemetry.proto.trace.v1.rs b/src/handlers/http/otel/opentelemetry.proto.trace.v1.rs new file mode 100644 index 000000000..ec8cd3d39 --- /dev/null +++ b/src/handlers/http/otel/opentelemetry.proto.trace.v1.rs @@ -0,0 +1,420 @@ +/* + * Parseable Server (C) 2022 - 2024 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +// This file was generated by protoc-gen-rust-protobuf. The file was edited after the generation. + // All the repeated fields were changed to Option>. + /// TracesData represents the traces data that can be stored in a persistent storage, +/// OR can be embedded by other protocols that transfer OTLP traces data but do +/// not implement the OTLP protocol. +/// +/// The main difference between this message and collector protocol is that +/// in this message there will not be any "control" or "metadata" specific to +/// OTLP protocol. +/// +/// When new fields are added into this message, the OTLP request MUST be updated +/// as well. + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug)] + #[serde(rename_all = "camelCase")] +pub struct TracesData { + /// An array of ResourceSpans. + /// For data coming from a single resource this array will typically contain + /// one element. Intermediary nodes that receive data from multiple origins + /// typically batch the data before forwarding further and in that case this + /// array will contain multiple elements. + pub resource_spans: Option>, +} +/// A collection of ScopeSpans from a Resource. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ResourceSpans { + /// The resource for the spans in this message. + /// If this field is not set then no resource info is known. + pub resource: Option, + /// A list of ScopeSpans that originate from a resource. + pub scope_spans: Option>, + /// The Schema URL, if known. This is the identifier of the Schema that the resource data + /// is recorded in. Notably, the last part of the URL path is the version number of the + /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see + /// + /// This schema_url applies to the data in the "resource" field. It does not apply + /// to the data in the "scope_spans" field which have their own schema_url field. + pub schema_url: Option, +} +/// A collection of Spans produced by an InstrumentationScope. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct ScopeSpans { + /// The instrumentation scope information for the spans in this message. + /// Semantically when InstrumentationScope isn't set, it is equivalent with + /// an empty instrumentation scope name (unknown). + pub scope: Option, + /// A list of Spans that originate from an instrumentation scope. + pub spans: Option>, + /// The Schema URL, if known. This is the identifier of the Schema that the span data + /// is recorded in. Notably, the last part of the URL path is the version number of the + /// schema: http\[s\]://server\[:port\]/path/. To learn more about Schema URL see + /// + /// This schema_url applies to all spans and span events in the "spans" field. + pub schema_url: Option, +} +/// A Span represents a single operation performed by a single component of the system. +/// +/// The next available field id is 17. +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Span { + /// A unique identifier for a trace. All spans from the same trace share + /// the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + /// of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + /// is zero-length and thus is also invalid). + /// + /// This field is required. + pub trace_id: Option, + /// A unique identifier for a span within a trace, assigned when the span + /// is created. The ID is an 8-byte array. An ID with all zeroes OR of length + /// other than 8 bytes is considered invalid (empty string in OTLP/JSON + /// is zero-length and thus is also invalid). + /// + /// This field is required. + pub span_id: Option, + /// trace_state conveys information about request position in multiple distributed tracing graphs. + /// It is a trace_state in w3c-trace-context format: + /// See also for more details about this field. + pub trace_state: Option, + /// The `span_id` of this span's parent span. If this is a root span, then this + /// field must be empty. The ID is an 8-byte array. + pub parent_span_id: Option, + /// Flags, a bit field. + /// + /// Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + /// Context specification. To read the 8-bit W3C trace flag, use + /// `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + /// + /// See for the flag definitions. + /// + /// Bits 8 and 9 represent the 3 states of whether a span's parent + /// is remote. The states are (unknown, is not remote, is remote). + /// To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + /// To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + /// + /// When creating span messages, if the message is logically forwarded from another source + /// with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + /// be copied as-is. If creating from a source that does not have an equivalent flags field + /// (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + /// be set to zero. + /// Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + /// + /// \[Optional\]. + pub flags: Option, + /// A description of the span's operation. + /// + /// For example, the name can be a qualified method name or a file name + /// and a line number where the operation is called. A best practice is to use + /// the same display name at the same call point in an application. + /// This makes it easier to correlate spans in different traces. + /// + /// This field is semantically required to be set to non-empty string. + /// Empty value is equivalent to an unknown span name. + /// + /// This field is required. + pub name: Option, + /// Distinguishes between spans generated in a particular context. For example, + /// two spans with the same name may be distinguished using `CLIENT` (caller) + /// and `SERVER` (callee) to identify queueing latency associated with the span. + pub kind: Option, + /// start_time_unix_nano is the start time of the span. On the client side, this is the time + /// kept by the local machine where the span execution starts. On the server side, this + /// is the time when the server's application handler starts running. + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + /// + /// This field is semantically required and it is expected that end_time >= start_time. + pub start_time_unix_nano: Option, + /// end_time_unix_nano is the end time of the span. On the client side, this is the time + /// kept by the local machine where the span execution ends. On the server side, this + /// is the time when the server application handler stops running. + /// Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + /// + /// This field is semantically required and it is expected that end_time >= start_time. + pub end_time_unix_nano: Option, + /// attributes is a collection of key/value pairs. Note, global attributes + /// like server name can be set using the resource API. Examples of attributes: + /// + /// "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + /// "/http/server_latency": 300 + /// "example.com/myattribute": true + /// "example.com/score": 10.239 + /// + /// The OpenTelemetry API specification further restricts the allowed value types: + /// + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// dropped_attributes_count is the number of attributes that were discarded. Attributes + /// can be discarded because their keys are too long or because there are too many + /// attributes. If this value is 0, then no attributes were dropped. + pub dropped_attributes_count: Option, + /// events is a collection of Event items. + pub events: Option>, + /// dropped_events_count is the number of dropped events. If the value is 0, then no + /// events were dropped. + pub dropped_events_count: Option, + /// links is a collection of Links, which are references from this span to a span + /// in the same or different trace. + pub links: Option>, + /// dropped_links_count is the number of dropped links after the maximum size was + /// enforced. If this value is 0, then no links were dropped. + pub dropped_links_count: Option, + /// An optional final status for this span. Semantically when Status isn't set, it means + /// span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + pub status: Option, +} +/// Nested message and enum types in `Span`. +pub mod span { + use serde::{Deserialize, Serialize}; + /// Event is a time-stamped annotation of the span, consisting of user-supplied + /// text description and key-value pairs. + #[derive(Serialize, Deserialize, Debug)] + pub struct Event { + /// time_unix_nano is the time the event occurred. + pub time_unix_nano: Option, + /// name of the event. + /// This field is semantically required to be set to non-empty string. + pub name: Option, + /// attributes is a collection of attribute key/value pairs on the event. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// dropped_attributes_count is the number of dropped attributes. If the value is 0, + /// then no attributes were dropped. + pub dropped_attributes_count: Option, + } + /// A pointer from the current span to another span in the same trace or in a + /// different trace. For example, this can be used in batching operations, + /// where a single batch handler processes multiple requests from different + /// traces or when the handler receives a request from a different project. + #[derive(Serialize, Deserialize, Debug)] + pub struct Link { + /// A unique identifier of a trace that this linked span is part of. The ID is a + /// 16-byte array. + pub trace_id: Option, + /// A unique identifier for the linked span. The ID is an 8-byte array. + pub span_id: Option, + /// The trace_state associated with the link. + pub trace_state: Option, + /// attributes is a collection of attribute key/value pairs on the link. + /// Attribute keys MUST be unique (it is not allowed to have more than one + /// attribute with the same key). + pub attributes: Option>, + /// dropped_attributes_count is the number of dropped attributes. If the value is 0, + /// then no attributes were dropped. + pub dropped_attributes_count: Option, + /// Flags, a bit field. + /// + /// Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + /// Context specification. To read the 8-bit W3C trace flag, use + /// `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + /// + /// See for the flag definitions. + /// + /// Bits 8 and 9 represent the 3 states of whether the link is remote. + /// The states are (unknown, is not remote, is remote). + /// To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + /// To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + /// + /// Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + /// When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + /// + /// \[Optional\]. + pub flags: Option, + } + /// SpanKind is the type of span. Can be used to specify additional relationships between spans + /// in addition to a parent/child relationship. + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum SpanKind { + /// Unspecified. Do NOT use as default. + /// Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + Unspecified = 0, + /// Indicates that the span represents an internal operation within an application, + /// as opposed to an operation happening at the boundaries. Default value. + Internal = 1, + /// Indicates that the span covers server-side handling of an RPC or other + /// remote network request. + Server = 2, + /// Indicates that the span describes a request to some remote service. + Client = 3, + /// Indicates that the span describes a producer sending a message to a broker. + /// Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + /// between producer and consumer spans. A PRODUCER span ends when the message was accepted + /// by the broker while the logical processing of the message might span a much longer time. + Producer = 4, + /// Indicates that the span describes consumer receiving a message from a broker. + /// Like the PRODUCER kind, there is often no direct critical path latency relationship + /// between producer and consumer spans. + Consumer = 5, + } + impl SpanKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "SPAN_KIND_UNSPECIFIED", + Self::Internal => "SPAN_KIND_INTERNAL", + Self::Server => "SPAN_KIND_SERVER", + Self::Client => "SPAN_KIND_CLIENT", + Self::Producer => "SPAN_KIND_PRODUCER", + Self::Consumer => "SPAN_KIND_CONSUMER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SPAN_KIND_UNSPECIFIED" => Some(Self::Unspecified), + "SPAN_KIND_INTERNAL" => Some(Self::Internal), + "SPAN_KIND_SERVER" => Some(Self::Server), + "SPAN_KIND_CLIENT" => Some(Self::Client), + "SPAN_KIND_PRODUCER" => Some(Self::Producer), + "SPAN_KIND_CONSUMER" => Some(Self::Consumer), + _ => None, + } + } + } +} +/// The Status type defines a logical error model that is suitable for different +/// programming environments, including REST APIs and RPC APIs. +#[derive(Serialize, Deserialize, Debug)] +pub struct Status { + /// A developer-facing human readable error message. + pub message: Option, + /// The status code. + pub code: Option, +} +/// Nested message and enum types in `Status`. +pub mod status { + /// For the semantics of status codes see + /// + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum StatusCode { + /// The default status. + Unset = 0, + /// The Span has been validated by an Application developer or Operator to + /// have completed successfully. + Ok = 1, + /// The Span contains an error. + Error = 2, + } + impl StatusCode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unset => "STATUS_CODE_UNSET", + Self::Ok => "STATUS_CODE_OK", + Self::Error => "STATUS_CODE_ERROR", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "STATUS_CODE_UNSET" => Some(Self::Unset), + "STATUS_CODE_OK" => Some(Self::Ok), + "STATUS_CODE_ERROR" => Some(Self::Error), + _ => None, + } + } + } +} +/// SpanFlags represents constants used to interpret the +/// Span.flags field, which is protobuf 'fixed32' type and is to +/// be used as bit-fields. Each non-zero value defined in this enum is +/// a bit-mask. To extract the bit-field, for example, use an +/// expression like: +/// +/// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +/// +/// See for the flag definitions. +/// +/// Note that Span flags were introduced in version 1.1 of the +/// OpenTelemetry protocol. Older Span producers do not set this +/// field, consequently consumers should not rely on the absence of a +/// particular flag bit to indicate the presence of a particular feature. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SpanFlags { + /// The zero value for the enum. Should not be used for comparisons. + /// Instead use bitwise "and" with the appropriate mask as shown above. + DoNotUse = 0, + /// Bits 0-7 are used for trace flags. + TraceFlagsMask = 255, + /// Bits 8 and 9 are used to indicate that the parent span or link span is remote. + /// Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + /// Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + ContextHasIsRemoteMask = 256, + ContextIsRemoteMask = 512, +} +impl SpanFlags { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::DoNotUse => "SPAN_FLAGS_DO_NOT_USE", + Self::TraceFlagsMask => "SPAN_FLAGS_TRACE_FLAGS_MASK", + Self::ContextHasIsRemoteMask => "SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK", + Self::ContextIsRemoteMask => "SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SPAN_FLAGS_DO_NOT_USE" => Some(Self::DoNotUse), + "SPAN_FLAGS_TRACE_FLAGS_MASK" => Some(Self::TraceFlagsMask), + "SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK" => Some(Self::ContextHasIsRemoteMask), + "SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK" => Some(Self::ContextIsRemoteMask), + _ => None, + } + } +} diff --git a/src/handlers/http/otel/opentelemetry/proto/common/v1/common.proto b/src/handlers/http/otel/opentelemetry/proto/common/v1/common.proto index f7ee8f265..59e348480 100644 --- a/src/handlers/http/otel/opentelemetry/proto/common/v1/common.proto +++ b/src/handlers/http/otel/opentelemetry/proto/common/v1/common.proto @@ -25,7 +25,7 @@ option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; // AnyValue is used to represent any type of attribute value. AnyValue may contain a // primitive value such as a string or integer or it may contain an arbitrary nested // object containing arrays, key-value lists and primitives. -message AnyValue { +message AnyValue { // The value is one of the listed fields. It is valid for all values to be unspecified // in which case this AnyValue is considered to be "empty". oneof value { diff --git a/src/handlers/http/otel/opentelemetry/proto/metrics/v1/metrics.proto b/src/handlers/http/otel/opentelemetry/proto/metrics/v1/metrics.proto new file mode 100644 index 000000000..00c5112ce --- /dev/null +++ b/src/handlers/http/otel/opentelemetry/proto/metrics/v1/metrics.proto @@ -0,0 +1,714 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.metrics.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Metrics.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.metrics.v1"; +option java_outer_classname = "MetricsProto"; +option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; + +// MetricsData represents the metrics data that can be stored in a persistent +// storage, OR can be embedded by other protocols that transfer OTLP metrics +// data but do not implement the OTLP protocol. +// +// MetricsData +// └─── ResourceMetrics +// ├── Resource +// ├── SchemaURL +// └── ScopeMetrics +// ├── Scope +// ├── SchemaURL +// └── Metric +// ├── Name +// ├── Description +// ├── Unit +// └── data +// ├── Gauge +// ├── Sum +// ├── Histogram +// ├── ExponentialHistogram +// └── Summary +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message MetricsData { + // An array of ResourceMetrics. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceMetrics resource_metrics = 1; +} + +// A collection of ScopeMetrics from a Resource. +message ResourceMetrics { + reserved 1000; + + // The resource for the metrics in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of metrics that originate from a resource. + repeated ScopeMetrics scope_metrics = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_metrics" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Metrics produced by an Scope. +message ScopeMetrics { + // The instrumentation scope information for the metrics in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of metrics that originate from an instrumentation library. + repeated Metric metrics = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the metric data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all metrics in the "metrics" field. + string schema_url = 3; +} + +// Defines a Metric which has one or more timeseries. The following is a +// brief summary of the Metric data model. For more details, see: +// +// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md +// +// The data model and relation between entities is shown in the +// diagram below. Here, "DataPoint" is the term used to refer to any +// one of the specific data point value types, and "points" is the term used +// to refer to any one of the lists of points contained in the Metric. +// +// - Metric is composed of a metadata and data. +// - Metadata part contains a name, description, unit. +// - Data is one of the possible types (Sum, Gauge, Histogram, Summary). +// - DataPoint contains timestamps, attributes, and one of the possible value type +// fields. +// +// Metric +// +------------+ +// |name | +// |description | +// |unit | +------------------------------------+ +// |data |---> |Gauge, Sum, Histogram, Summary, ... | +// +------------+ +------------------------------------+ +// +// Data [One of Gauge, Sum, Histogram, Summary, ...] +// +-----------+ +// |... | // Metadata about the Data. +// |points |--+ +// +-----------+ | +// | +---------------------------+ +// | |DataPoint 1 | +// v |+------+------+ +------+ | +// +-----+ ||label |label |...|label | | +// | 1 |-->||value1|value2|...|valueN| | +// +-----+ |+------+------+ +------+ | +// | . | |+-----+ | +// | . | ||value| | +// | . | |+-----+ | +// | . | +---------------------------+ +// | . | . +// | . | . +// | . | . +// | . | +---------------------------+ +// | . | |DataPoint M | +// +-----+ |+------+------+ +------+ | +// | M |-->||label |label |...|label | | +// +-----+ ||value1|value2|...|valueN| | +// |+------+------+ +------+ | +// |+-----+ | +// ||value| | +// |+-----+ | +// +---------------------------+ +// +// Each distinct type of DataPoint represents the output of a specific +// aggregation function, the result of applying the DataPoint's +// associated function of to one or more measurements. +// +// All DataPoint types have three common fields: +// - Attributes includes key-value pairs associated with the data point +// - TimeUnixNano is required, set to the end time of the aggregation +// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints +// having an AggregationTemporality field, as discussed below. +// +// Both TimeUnixNano and StartTimeUnixNano values are expressed as +// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. +// +// # TimeUnixNano +// +// This field is required, having consistent interpretation across +// DataPoint types. TimeUnixNano is the moment corresponding to when +// the data point's aggregate value was captured. +// +// Data points with the 0 value for TimeUnixNano SHOULD be rejected +// by consumers. +// +// # StartTimeUnixNano +// +// StartTimeUnixNano in general allows detecting when a sequence of +// observations is unbroken. This field indicates to consumers the +// start time for points with cumulative and delta +// AggregationTemporality, and it should be included whenever possible +// to support correct rate calculation. Although it may be omitted +// when the start time is truly unknown, setting StartTimeUnixNano is +// strongly encouraged. +message Metric { + reserved 4, 6, 8; + + // name of the metric. + string name = 1; + + // description of the metric, which can be used in documentation. + string description = 2; + + // unit in which the metric value is reported. Follows the format + // described by http://unitsofmeasure.org/ucum.html. + string unit = 3; + + // Data determines the aggregation type (if any) of the metric, what is the + // reported value type for the data points, as well as the relatationship to + // the time interval over which they are reported. + oneof data { + Gauge gauge = 5; + Sum sum = 7; + Histogram histogram = 9; + ExponentialHistogram exponential_histogram = 10; + Summary summary = 11; + } + + // Additional metadata attributes that describe the metric. [Optional]. + // Attributes are non-identifying. + // Consumers SHOULD NOT need to be aware of these attributes. + // These attributes MAY be used to encode information allowing + // for lossless roundtrip translation to / from another data model. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue metadata = 12; +} + +// Gauge represents the type of a scalar metric that always exports the +// "current value" for every data point. It should be used for an "unknown" +// aggregation. +// +// A Gauge does not support different aggregation temporalities. Given the +// aggregation is unknown, points cannot be combined using the same +// aggregation, regardless of aggregation temporalities. Therefore, +// AggregationTemporality is not included. Consequently, this also means +// "StartTimeUnixNano" is ignored for all data points. +message Gauge { + repeated NumberDataPoint data_points = 1; +} + +// Sum represents the type of a scalar metric that is calculated as a sum of all +// reported measurements over a time interval. +message Sum { + repeated NumberDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; + + // If "true" means that the sum is monotonic. + bool is_monotonic = 3; +} + +// Histogram represents the type of a metric that is calculated by aggregating +// as a Histogram of all reported measurements over a time interval. +message Histogram { + repeated HistogramDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// ExponentialHistogram represents the type of a metric that is calculated by aggregating +// as a ExponentialHistogram of all reported double measurements over a time interval. +message ExponentialHistogram { + repeated ExponentialHistogramDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// Summary metric data are used to convey quantile summaries, +// a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) +// and OpenMetrics (see: https://github.com/OpenObservability/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) +// data type. These data points cannot always be merged in a meaningful way. +// While they can be useful in some applications, histogram data points are +// recommended for new applications. +// Summary metrics do not have an aggregation temporality field. This is +// because the count and sum fields of a SummaryDataPoint are assumed to be +// cumulative values. +message Summary { + repeated SummaryDataPoint data_points = 1; +} + +// AggregationTemporality defines how a metric aggregator reports aggregated +// values. It describes how those values relate to the time interval over +// which they are aggregated. +enum AggregationTemporality { + // UNSPECIFIED is the default AggregationTemporality, it MUST not be used. + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + + // DELTA is an AggregationTemporality for a metric aggregator which reports + // changes since last report time. Successive metrics contain aggregation of + // values from continuous and non-overlapping intervals. + // + // The values for a DELTA metric are based only on the time interval + // associated with one measurement cycle. There is no dependency on + // previous measurements like is the case for CUMULATIVE metrics. + // + // For example, consider a system measuring the number of requests that + // it receives and reports the sum of these requests every second as a + // DELTA metric: + // + // 1. The system starts receiving at time=t_0. + // 2. A request is received, the system measures 1 request. + // 3. A request is received, the system measures 1 request. + // 4. A request is received, the system measures 1 request. + // 5. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+1 with a value of 3. + // 6. A request is received, the system measures 1 request. + // 7. A request is received, the system measures 1 request. + // 8. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0+1 to + // t_0+2 with a value of 2. + AGGREGATION_TEMPORALITY_DELTA = 1; + + // CUMULATIVE is an AggregationTemporality for a metric aggregator which + // reports changes since a fixed start time. This means that current values + // of a CUMULATIVE metric depend on all previous measurements since the + // start time. Because of this, the sender is required to retain this state + // in some form. If this state is lost or invalidated, the CUMULATIVE metric + // values MUST be reset and a new fixed start time following the last + // reported measurement time sent MUST be used. + // + // For example, consider a system measuring the number of requests that + // it receives and reports the sum of these requests every second as a + // CUMULATIVE metric: + // + // 1. The system starts receiving at time=t_0. + // 2. A request is received, the system measures 1 request. + // 3. A request is received, the system measures 1 request. + // 4. A request is received, the system measures 1 request. + // 5. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+1 with a value of 3. + // 6. A request is received, the system measures 1 request. + // 7. A request is received, the system measures 1 request. + // 8. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+2 with a value of 5. + // 9. The system experiences a fault and loses state. + // 10. The system recovers and resumes receiving at time=t_1. + // 11. A request is received, the system measures 1 request. + // 12. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_1 to + // t_0+1 with a value of 1. + // + // Note: Even though, when reporting changes since last report time, using + // CUMULATIVE is valid, it is not recommended. This may cause problems for + // systems that do not use start_time to determine when the aggregation + // value was reset (e.g. Prometheus). + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +// DataPointFlags is defined as a protobuf 'uint32' type and is to be used as a +// bit-field representing 32 distinct boolean flags. Each flag defined in this +// enum is a bit-mask. To test the presence of a single flag in the flags of +// a data point, for example, use an expression like: +// +// (point.flags & DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) == DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK +// +enum DataPointFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + DATA_POINT_FLAGS_DO_NOT_USE = 0; + + // This DataPoint is valid but has no recorded value. This value + // SHOULD be used to reflect explicitly missing data in a series, as + // for an equivalent to the Prometheus "staleness marker". + DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1; + + // Bits 2-31 are reserved for future use. +} + +// NumberDataPoint is a single data point in a timeseries that describes the +// time-varying scalar value of a metric. +message NumberDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // The value itself. A point is considered invalid when one of the recognized + // value fields is not present inside this oneof. + oneof value { + double as_double = 4; + sfixed64 as_int = 6; + } + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 5; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 8; +} + +// HistogramDataPoint is a single data point in a timeseries that describes the +// time-varying values of a Histogram. A Histogram contains summary statistics +// for a population of values, it may optionally contain the distribution of +// those values across a set of buckets. +// +// If the histogram contains the distribution of values, then both +// "explicit_bounds" and "bucket counts" fields must be defined. +// If the histogram does not contain the distribution of values, then both +// "explicit_bounds" and "bucket_counts" must be omitted and only "count" and +// "sum" are known. +message HistogramDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be non-negative. This + // value must be equal to the sum of the "count" fields in buckets if a + // histogram is provided. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + optional double sum = 5; + + // bucket_counts is an optional field contains the count values of histogram + // for each bucket. + // + // The sum of the bucket_counts must equal the value in the count field. + // + // The number of elements in bucket_counts array must be by one greater than + // the number of elements in explicit_bounds array. + repeated fixed64 bucket_counts = 6; + + // explicit_bounds specifies buckets with explicitly defined bounds for values. + // + // The boundaries for bucket at index i are: + // + // (-infinity, explicit_bounds[i]] for i == 0 + // (explicit_bounds[i-1], explicit_bounds[i]] for 0 < i < size(explicit_bounds) + // (explicit_bounds[i-1], +infinity) for i == size(explicit_bounds) + // + // The values in the explicit_bounds array must be strictly increasing. + // + // Histogram buckets are inclusive of their upper boundary, except the last + // bucket where the boundary is at infinity. This format is intentionally + // compatible with the OpenMetrics histogram definition. + repeated double explicit_bounds = 7; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 8; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 10; + + // min is the minimum value over (start_time, end_time]. + optional double min = 11; + + // max is the maximum value over (start_time, end_time]. + optional double max = 12; +} + +// ExponentialHistogramDataPoint is a single data point in a timeseries that describes the +// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains +// summary statistics for a population of values, it may optionally contain the +// distribution of those values across a set of buckets. +// +message ExponentialHistogramDataPoint { + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be + // non-negative. This value must be equal to the sum of the "bucket_counts" + // values in the positive and negative Buckets plus the "zero_count" field. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + optional double sum = 5; + + // scale describes the resolution of the histogram. Boundaries are + // located at powers of the base, where: + // + // base = (2^(2^-scale)) + // + // The histogram bucket identified by `index`, a signed integer, + // contains values that are greater than (base^index) and + // less than or equal to (base^(index+1)). + // + // The positive and negative ranges of the histogram are expressed + // separately. Negative values are mapped by their absolute value + // into the negative range using the same scale as the positive range. + // + // scale is not restricted by the protocol, as the permissible + // values depend on the range of the data. + sint32 scale = 6; + + // zero_count is the count of values that are either exactly zero or + // within the region considered zero by the instrumentation at the + // tolerated degree of precision. This bucket stores values that + // cannot be expressed using the standard exponential formula as + // well as values that have been rounded to zero. + // + // Implementations MAY consider the zero bucket to have probability + // mass equal to (zero_count / count). + fixed64 zero_count = 7; + + // positive carries the positive range of exponential bucket counts. + Buckets positive = 8; + + // negative carries the negative range of exponential bucket counts. + Buckets negative = 9; + + // Buckets are a set of bucket counts, encoded in a contiguous array + // of counts. + message Buckets { + // Offset is the bucket index of the first entry in the bucket_counts array. + // + // Note: This uses a varint encoding as a simple form of compression. + sint32 offset = 1; + + // bucket_counts is an array of count values, where bucket_counts[i] carries + // the count of the bucket at index (offset+i). bucket_counts[i] is the count + // of values greater than base^(offset+i) and less than or equal to + // base^(offset+i+1). + // + // Note: By contrast, the explicit HistogramDataPoint uses + // fixed64. This field is expected to have many buckets, + // especially zeros, so uint64 has been selected to ensure + // varint encoding. + repeated uint64 bucket_counts = 2; + } + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 10; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 11; + + // min is the minimum value over (start_time, end_time]. + optional double min = 12; + + // max is the maximum value over (start_time, end_time]. + optional double max = 13; + + // ZeroThreshold may be optionally set to convey the width of the zero + // region. Where the zero region is defined as the closed interval + // [-ZeroThreshold, ZeroThreshold]. + // When ZeroThreshold is 0, zero count bucket stores values that cannot be + // expressed using the standard exponential formula as well as values that + // have been rounded to zero. + double zero_threshold = 14; +} + +// SummaryDataPoint is a single data point in a timeseries that describes the +// time-varying values of a Summary metric. The count and sum fields represent +// cumulative values. +message SummaryDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list may be empty (may contain 0 elements). + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be non-negative. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary + double sum = 5; + + // Represents the value at a given quantile of a distribution. + // + // To record Min and Max values following conventions are used: + // - The 1.0 quantile is equivalent to the maximum value observed. + // - The 0.0 quantile is equivalent to the minimum value observed. + // + // See the following issue for more context: + // https://github.com/open-telemetry/opentelemetry-proto/issues/125 + message ValueAtQuantile { + // The quantile of a distribution. Must be in the interval + // [0.0, 1.0]. + double quantile = 1; + + // The value at the given quantile of a distribution. + // + // Quantile values must NOT be negative. + double value = 2; + } + + // (Optional) list of values at different quantiles of the distribution calculated + // from the current snapshot. The quantiles must be strictly increasing. + repeated ValueAtQuantile quantile_values = 6; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 8; +} + +// A representation of an exemplar, which is a sample input measurement. +// Exemplars also hold information about the environment when the measurement +// was recorded, for example the span and trace ID of the active span when the +// exemplar was recorded. +message Exemplar { + reserved 1; + + // The set of key/value pairs that were filtered out by the aggregator, but + // recorded alongside the original measurement. Only key/value pairs that were + // filtered out by the aggregator should be included + repeated opentelemetry.proto.common.v1.KeyValue filtered_attributes = 7; + + // time_unix_nano is the exact time when this exemplar was recorded + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 2; + + // The value of the measurement that was recorded. An exemplar is + // considered invalid when one of the recognized value fields is not present + // inside this oneof. + oneof value { + double as_double = 3; + sfixed64 as_int = 6; + } + + // (Optional) Span ID of the exemplar trace. + // span_id may be missing if the measurement is not recorded inside a trace + // or if the trace is not sampled. + bytes span_id = 4; + + // (Optional) Trace ID of the exemplar trace. + // trace_id may be missing if the measurement is not recorded inside a trace + // or if the trace is not sampled. + bytes trace_id = 5; +} diff --git a/src/handlers/http/otel/opentelemetry/proto/trace/v1/trace.proto b/src/handlers/http/otel/opentelemetry/proto/trace/v1/trace.proto new file mode 100644 index 000000000..24442853e --- /dev/null +++ b/src/handlers/http/otel/opentelemetry/proto/trace/v1/trace.proto @@ -0,0 +1,357 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.trace.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Trace.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.trace.v1"; +option java_outer_classname = "TraceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/trace/v1"; + +// TracesData represents the traces data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP traces data but do +// not implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message TracesData { + // An array of ResourceSpans. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceSpans resource_spans = 1; +} + +// A collection of ScopeSpans from a Resource. +message ResourceSpans { + reserved 1000; + + // The resource for the spans in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeSpans that originate from a resource. + repeated ScopeSpans scope_spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_spans" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Spans produced by an InstrumentationScope. +message ScopeSpans { + // The instrumentation scope information for the spans in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Spans that originate from an instrumentation scope. + repeated Span spans = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the span data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all spans and span events in the "spans" field. + string schema_url = 3; +} + +// A Span represents a single operation performed by a single component of the system. +// +// The next available field id is 17. +message Span { + // A unique identifier for a trace. All spans from the same trace share + // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR + // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes trace_id = 1; + + // A unique identifier for a span within a trace, assigned when the span + // is created. The ID is an 8-byte array. An ID with all zeroes OR of length + // other than 8 bytes is considered invalid (empty string in OTLP/JSON + // is zero-length and thus is also invalid). + // + // This field is required. + bytes span_id = 2; + + // trace_state conveys information about request position in multiple distributed tracing graphs. + // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header + // See also https://github.com/w3c/distributed-tracing for more details about this field. + string trace_state = 3; + + // The `span_id` of this span's parent span. If this is a root span, then this + // field must be empty. The ID is an 8-byte array. + bytes parent_span_id = 4; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether a span's parent + // is remote. The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // When creating span messages, if the message is logically forwarded from another source + // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD + // be copied as-is. If creating from a source that does not have an equivalent flags field + // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST + // be set to zero. + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // + // [Optional]. + fixed32 flags = 16; + + // A description of the span's operation. + // + // For example, the name can be a qualified method name or a file name + // and a line number where the operation is called. A best practice is to use + // the same display name at the same call point in an application. + // This makes it easier to correlate spans in different traces. + // + // This field is semantically required to be set to non-empty string. + // Empty value is equivalent to an unknown span name. + // + // This field is required. + string name = 5; + + // SpanKind is the type of span. Can be used to specify additional relationships between spans + // in addition to a parent/child relationship. + enum SpanKind { + // Unspecified. Do NOT use as default. + // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED. + SPAN_KIND_UNSPECIFIED = 0; + + // Indicates that the span represents an internal operation within an application, + // as opposed to an operation happening at the boundaries. Default value. + SPAN_KIND_INTERNAL = 1; + + // Indicates that the span covers server-side handling of an RPC or other + // remote network request. + SPAN_KIND_SERVER = 2; + + // Indicates that the span describes a request to some remote service. + SPAN_KIND_CLIENT = 3; + + // Indicates that the span describes a producer sending a message to a broker. + // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship + // between producer and consumer spans. A PRODUCER span ends when the message was accepted + // by the broker while the logical processing of the message might span a much longer time. + SPAN_KIND_PRODUCER = 4; + + // Indicates that the span describes consumer receiving a message from a broker. + // Like the PRODUCER kind, there is often no direct critical path latency relationship + // between producer and consumer spans. + SPAN_KIND_CONSUMER = 5; + } + + // Distinguishes between spans generated in a particular context. For example, + // two spans with the same name may be distinguished using `CLIENT` (caller) + // and `SERVER` (callee) to identify queueing latency associated with the span. + SpanKind kind = 6; + + // start_time_unix_nano is the start time of the span. On the client side, this is the time + // kept by the local machine where the span execution starts. On the server side, this + // is the time when the server's application handler starts running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 start_time_unix_nano = 7; + + // end_time_unix_nano is the end time of the span. On the client side, this is the time + // kept by the local machine where the span execution ends. On the server side, this + // is the time when the server application handler stops running. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 end_time_unix_nano = 8; + + // attributes is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "example.com/myattribute": true + // "example.com/score": 10.239 + // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 10; + + // Event is a time-stamped annotation of the span, consisting of user-supplied + // text description and key-value pairs. + message Event { + // time_unix_nano is the time the event occurred. + fixed64 time_unix_nano = 1; + + // name of the event. + // This field is semantically required to be set to non-empty string. + string name = 2; + + // attributes is a collection of attribute key/value pairs on the event. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 4; + } + + // events is a collection of Event items. + repeated Event events = 11; + + // dropped_events_count is the number of dropped events. If the value is 0, then no + // events were dropped. + uint32 dropped_events_count = 12; + + // A pointer from the current span to another span in the same trace or in a + // different trace. For example, this can be used in batching operations, + // where a single batch handler processes multiple requests from different + // traces or when the handler receives a request from a different project. + message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; + + // The trace_state associated with the link. + string trace_state = 3; + + // attributes is a collection of attribute key/value pairs on the link. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + + // dropped_attributes_count is the number of dropped attributes. If the value is 0, + // then no attributes were dropped. + uint32 dropped_attributes_count = 5; + + // Flags, a bit field. + // + // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace + // Context specification. To read the 8-bit W3C trace flag, use + // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`. + // + // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. + // + // Bits 8 and 9 represent the 3 states of whether the link is remote. + // The states are (unknown, is not remote, is remote). + // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`. + // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`. + // + // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero. + // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero. + // + // [Optional]. + fixed32 flags = 6; + } + + // links is a collection of Links, which are references from this span to a span + // in the same or different trace. + repeated Link links = 13; + + // dropped_links_count is the number of dropped links after the maximum size was + // enforced. If this value is 0, then no links were dropped. + uint32 dropped_links_count = 14; + + // An optional final status for this span. Semantically when Status isn't set, it means + // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0). + Status status = 15; +} + +// The Status type defines a logical error model that is suitable for different +// programming environments, including REST APIs and RPC APIs. +message Status { + reserved 1; + + // A developer-facing human readable error message. + string message = 2; + + // For the semantics of status codes see + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status + enum StatusCode { + // The default status. + STATUS_CODE_UNSET = 0; + // The Span has been validated by an Application developer or Operator to + // have completed successfully. + STATUS_CODE_OK = 1; + // The Span contains an error. + STATUS_CODE_ERROR = 2; + }; + + // The status code. + StatusCode code = 3; +} + +// SpanFlags represents constants used to interpret the +// Span.flags field, which is protobuf 'fixed32' type and is to +// be used as bit-fields. Each non-zero value defined in this enum is +// a bit-mask. To extract the bit-field, for example, use an +// expression like: +// +// (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK) +// +// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions. +// +// Note that Span flags were introduced in version 1.1 of the +// OpenTelemetry protocol. Older Span producers do not set this +// field, consequently consumers should not rely on the absence of a +// particular flag bit to indicate the presence of a particular feature. +enum SpanFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + SPAN_FLAGS_DO_NOT_USE = 0; + + // Bits 0-7 are used for trace flags. + SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; + + // Bits 8 and 9 are used to indicate that the parent span or link span is remote. + // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known. + // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote. + SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; + SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; + + // Bits 10-31 are reserved for future use. +} diff --git a/src/handlers/http/otel/proto.rs b/src/handlers/http/otel/proto.rs index 9322bfcc5..d48d453cb 100644 --- a/src/handlers/http/otel/proto.rs +++ b/src/handlers/http/otel/proto.rs @@ -36,3 +36,17 @@ pub mod resource { include!("opentelemetry.proto.resource.v1.rs"); } } + +/// Generated types used in metrics. +pub mod metrics { + pub mod v1 { + include!("opentelemetry.proto.metrics.v1.rs"); + } +} + +/// Generated types used in traces. +pub mod trace { + pub mod v1 { + include!("opentelemetry.proto.trace.v1.rs"); + } +} diff --git a/src/handlers/http/otel_logs.rs b/src/handlers/http/otel_logs.rs new file mode 100644 index 000000000..8ffc7c4e0 --- /dev/null +++ b/src/handlers/http/otel_logs.rs @@ -0,0 +1,169 @@ +/* + * Parseable Server (C) 2022 - 2024 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +use std::collections::BTreeMap; + +use crate::handlers::http::otel::proto::logs::v1::LogRecord; +use crate::handlers::http::otel::proto::logs::v1::LogRecordFlags; +use crate::handlers::http::otel::proto::logs::v1::LogsData; +use crate::handlers::http::otel::proto::logs::v1::SeverityNumber; +use bytes::Bytes; +use serde_json::Value; + +use super::otel::collect_json_from_values; +use super::otel::insert_attributes; +use super::otel::insert_if_some; +use super::otel::insert_number_if_some; +use super::otel::proto::logs::v1::ScopeLogs; + +fn flatten_severity(severity_number: &Option) -> BTreeMap { + let mut severity_json: BTreeMap = BTreeMap::new(); + insert_number_if_some( + &mut severity_json, + "severity_number", + &severity_number.map(|f| f as f64), + ); + if let Some(severity_number) = severity_number { + insert_if_some( + &mut severity_json, + "severity_text", + &Some(SeverityNumber::as_str_name(*severity_number)), + ); + } + severity_json +} + +fn flatten_flags(flags: &Option) -> BTreeMap { + let mut flags_json: BTreeMap = BTreeMap::new(); + insert_number_if_some(&mut flags_json, "flags_number", &flags.map(|f| f as f64)); + if let Some(flags) = flags { + insert_if_some( + &mut flags_json, + "flags_string", + &Some(LogRecordFlags::as_str_name(*flags)), + ); + } + flags_json +} +//flatten log record and all its attributes to create one record for each log record +pub fn flatten_log_record(log_record: &LogRecord) -> BTreeMap { + let mut log_record_json: BTreeMap = BTreeMap::new(); + insert_if_some( + &mut log_record_json, + "time_unix_nano", + &log_record.time_unix_nano, + ); + insert_if_some( + &mut log_record_json, + "observed_time_unix_nano", + &log_record.observed_time_unix_nano, + ); + + log_record_json.extend(flatten_severity(&log_record.severity_number)); + + if log_record.body.is_some() { + let body = &log_record.body; + let body_json = collect_json_from_values(body, &"body".to_string()); + for key in body_json.keys() { + log_record_json.insert(key.to_owned(), body_json[key].to_owned()); + } + } + insert_attributes(&mut log_record_json, &log_record.attributes); + insert_number_if_some( + &mut log_record_json, + "log_record_dropped_attributes_count", + &log_record.dropped_attributes_count.map(|f| f as f64), + ); + + log_record_json.extend(flatten_flags(&log_record.flags)); + insert_if_some(&mut log_record_json, "span_id", &log_record.span_id); + insert_if_some(&mut log_record_json, "trace_id", &log_record.trace_id); + + log_record_json +} + +//flatten otel logs +fn flatten_scope_log(scope_log: &ScopeLogs) -> Vec> { + let mut vec_scope_log_json = Vec::new(); + let mut scope_log_json = BTreeMap::new(); + + if let Some(scope) = &scope_log.scope { + insert_if_some(&mut scope_log_json, "scope_name", &scope.name); + insert_if_some(&mut scope_log_json, "scope_version", &scope.version); + insert_attributes(&mut scope_log_json, &scope.attributes); + insert_number_if_some( + &mut scope_log_json, + "scope_dropped_attributes_count", + &scope.dropped_attributes_count.map(|f| f as f64), + ); + } + + if let Some(schema_url) = &scope_log.schema_url { + scope_log_json.insert( + "scope_log_schema_url".to_string(), + Value::String(schema_url.clone()), + ); + } + + for log_record in &scope_log.log_records { + let log_record_json = flatten_log_record(log_record); + let mut combined_json = scope_log_json.clone(); + combined_json.extend(log_record_json); + vec_scope_log_json.push(combined_json); + } + + vec_scope_log_json +} + +pub fn flatten_otel_logs(body: &Bytes) -> Vec> { + let body_str = std::str::from_utf8(body).unwrap(); + let message: LogsData = serde_json::from_str(body_str).unwrap(); + let mut vec_otel_json = Vec::new(); + + if let Some(records) = &message.resource_logs { + for record in records { + let mut resource_log_json = BTreeMap::new(); + + if let Some(resource) = &record.resource { + insert_attributes(&mut resource_log_json, &resource.attributes); + insert_number_if_some( + &mut resource_log_json, + "resource_dropped_attributes_count", + &resource.dropped_attributes_count.map(|f| f as f64), + ); + } + + let mut vec_resource_logs_json = Vec::new(); + if let Some(scope_logs) = &record.scope_logs { + for scope_log in scope_logs { + vec_resource_logs_json.extend(flatten_scope_log(scope_log)); + } + } + + insert_if_some(&mut resource_log_json, "schema_url", &record.schema_url); + + for resource_logs_json in &mut vec_resource_logs_json { + resource_logs_json.extend(resource_log_json.clone()); + } + + vec_otel_json.extend(vec_resource_logs_json); + } + } + + vec_otel_json +} diff --git a/src/handlers/http/otel_metrics.rs b/src/handlers/http/otel_metrics.rs new file mode 100644 index 000000000..cce83c517 --- /dev/null +++ b/src/handlers/http/otel_metrics.rs @@ -0,0 +1,445 @@ +/* + * Parseable Server (C) 2022 - 2024 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +use std::collections::BTreeMap; + +use bytes::Bytes; +use serde_json::Value; + +use super::otel::{ + insert_attributes, insert_bool_if_some, insert_if_some, insert_number_if_some, + proto::metrics::v1::{ + exponential_histogram_data_point::Buckets, Exemplar, ExponentialHistogram, Gauge, + Histogram, Metric, MetricsData, NumberDataPoint, Sum, Summary, + }, +}; + +fn flatten_exemplar(exemplars: &[Exemplar]) -> BTreeMap { + let mut exemplar_json = BTreeMap::new(); + for exemplar in exemplars { + insert_attributes(&mut exemplar_json, &exemplar.filtered_attributes); + insert_if_some( + &mut exemplar_json, + "exemplar_time_unix_nano", + &exemplar.time_unix_nano, + ); + insert_if_some(&mut exemplar_json, "exemplar_span_id", &exemplar.span_id); + insert_if_some(&mut exemplar_json, "exemplar_trace_id", &exemplar.trace_id); + insert_number_if_some( + &mut exemplar_json, + "exemplar_as_double", + &exemplar.as_double, + ); + insert_if_some(&mut exemplar_json, "exemplar_as_int", &exemplar.as_int); + } + exemplar_json +} + +fn flatten_number_data_points(data_points: &[NumberDataPoint]) -> Vec> { + data_points + .iter() + .map(|data_point| { + let mut data_point_json = BTreeMap::new(); + insert_attributes(&mut data_point_json, &data_point.attributes); + insert_if_some( + &mut data_point_json, + "start_time_unix_nano", + &data_point.start_time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "time_unix_nano", + &data_point.time_unix_nano, + ); + insert_number_if_some( + &mut data_point_json, + "data_point_as_double", + &data_point.as_double, + ); + insert_if_some( + &mut data_point_json, + "data_point_as_int", + &data_point.as_int, + ); + if let Some(exemplars) = &data_point.exemplars { + let exemplar_json = flatten_exemplar(exemplars); + for (key, value) in exemplar_json { + data_point_json.insert(key, value); + } + } + data_point_json + }) + .collect() +} + +fn flatten_gauge(gauge: &Gauge) -> Vec> { + let mut vec_gauge_json = Vec::new(); + if let Some(data_points) = &gauge.data_points { + let data_points_json = flatten_number_data_points(data_points); + for data_point_json in data_points_json { + let mut gauge_json = BTreeMap::new(); + for (key, value) in &data_point_json { + gauge_json.insert(format!("gauge_{}", key), value.clone()); + } + vec_gauge_json.push(gauge_json); + } + } + vec_gauge_json +} + +fn flatten_sum(sum: &Sum) -> Vec> { + let mut vec_sum_json = Vec::new(); + if let Some(data_points) = &sum.data_points { + let data_points_json = flatten_number_data_points(data_points); + for data_point_json in data_points_json { + let mut sum_json = BTreeMap::new(); + for (key, value) in &data_point_json { + sum_json.insert(format!("sum_{}", key), value.clone()); + } + vec_sum_json.push(sum_json); + } + let mut sum_json = BTreeMap::new(); + sum_json.extend(flatten_aggregation_temporality( + &sum.aggregation_temporality, + )); + insert_bool_if_some(&mut sum_json, "sum_is_monotonic", &sum.is_monotonic); + for data_point_json in &mut vec_sum_json { + for (key, value) in &sum_json { + data_point_json.insert(key.clone(), value.clone()); + } + } + } + vec_sum_json +} + +fn flatten_histogram(histogram: &Histogram) -> Vec> { + let mut data_points_json = Vec::new(); + if let Some(histogram_data_points) = &histogram.data_points { + for data_point in histogram_data_points { + let mut data_point_json = BTreeMap::new(); + insert_attributes(&mut data_point_json, &data_point.attributes); + insert_if_some( + &mut data_point_json, + "histogram_start_time_unix_nano", + &data_point.start_time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "histogram_time_unix_nano", + &data_point.time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "histogram_data_point_count", + &data_point.count, + ); + insert_number_if_some( + &mut data_point_json, + "histogram_data_point_sum", + &data_point.sum, + ); + if let Some(bucket_counts) = &data_point.bucket_counts { + for (index, bucket_count) in bucket_counts.iter().enumerate() { + data_point_json.insert( + format!("histogram_data_point_bucket_count_{}", index + 1), + Value::String(bucket_count.to_string()), + ); + } + } + if let Some(explicit_bounds) = &data_point.explicit_bounds { + for (index, explicit_bound) in explicit_bounds.iter().enumerate() { + data_point_json.insert( + format!("histogram_data_point_explicit_bound_{}", index + 1), + Value::String(explicit_bound.to_string()), + ); + } + } + if let Some(exemplars) = &data_point.exemplars { + let exemplar_json = flatten_exemplar(exemplars); + for (key, value) in exemplar_json { + data_point_json.insert(format!("histogram_{}", key), value); + } + } + data_points_json.push(data_point_json); + } + } + let mut histogram_json = BTreeMap::new(); + histogram_json.extend(flatten_aggregation_temporality( + &histogram.aggregation_temporality, + )); + for data_point_json in &mut data_points_json { + for (key, value) in &histogram_json { + data_point_json.insert(key.clone(), value.clone()); + } + } + data_points_json +} + +fn flatten_buckets(bucket: &Buckets) -> BTreeMap { + let mut bucket_json = BTreeMap::new(); + insert_number_if_some(&mut bucket_json, "offset", &bucket.offset.map(|v| v as f64)); + if let Some(bucket_counts) = &bucket.bucket_counts { + for (index, bucket_count) in bucket_counts.iter().enumerate() { + bucket_json.insert( + format!("bucket_count_{}", index + 1), + Value::String(bucket_count.to_string()), + ); + } + } + bucket_json +} + +fn flatten_exp_histogram(exp_histogram: &ExponentialHistogram) -> Vec> { + let mut data_points_json = Vec::new(); + if let Some(exp_histogram_data_points) = &exp_histogram.data_points { + for data_point in exp_histogram_data_points { + let mut data_point_json = BTreeMap::new(); + insert_attributes(&mut data_point_json, &data_point.attributes); + insert_if_some( + &mut data_point_json, + "exponential_histogram_start_time_unix_nano", + &data_point.start_time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "exponential_histogram_time_unix_nano", + &data_point.time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "exponential_histogram_data_point_count", + &data_point.count, + ); + insert_number_if_some( + &mut data_point_json, + "exponential_histogram_data_point_sum", + &data_point.sum, + ); + insert_number_if_some( + &mut data_point_json, + "exponential_histogram_data_point_scale", + &data_point.scale.map(|v| v as f64), + ); + insert_number_if_some( + &mut data_point_json, + "exponential_histogram_data_point_zero_count", + &data_point.zero_count.map(|v| v as f64), + ); + if let Some(positive) = &data_point.positive { + let positive_json = flatten_buckets(positive); + for (key, value) in positive_json { + data_point_json + .insert(format!("exponential_histogram_positive_{}", key), value); + } + } + if let Some(negative) = &data_point.negative { + let negative_json = flatten_buckets(negative); + for (key, value) in negative_json { + data_point_json + .insert(format!("exponential_histogram_negative_{}", key), value); + } + } + if let Some(exemplars) = &data_point.exemplars { + let exemplar_json = flatten_exemplar(exemplars); + for (key, value) in exemplar_json { + data_point_json.insert(format!("exponential_histogram_{}", key), value); + } + } + data_points_json.push(data_point_json); + } + } + let mut exp_histogram_json = BTreeMap::new(); + exp_histogram_json.extend(flatten_aggregation_temporality( + &exp_histogram.aggregation_temporality, + )); + for data_point_json in &mut data_points_json { + for (key, value) in &exp_histogram_json { + data_point_json.insert(key.clone(), value.clone()); + } + } + data_points_json +} + +fn flatten_summary(summary: &Summary) -> Vec> { + let mut data_points_json = Vec::new(); + if let Some(summary_data_points) = &summary.data_points { + for data_point in summary_data_points { + let mut data_point_json = BTreeMap::new(); + insert_attributes(&mut data_point_json, &data_point.attributes); + insert_if_some( + &mut data_point_json, + "summary_start_time_unix_nano", + &data_point.start_time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "summary_time_unix_nano", + &data_point.time_unix_nano, + ); + insert_if_some( + &mut data_point_json, + "summary_data_point_count", + &data_point.count, + ); + insert_number_if_some( + &mut data_point_json, + "summary_data_point_sum", + &data_point.sum, + ); + if let Some(quantile_values) = &data_point.quantile_values { + for (index, quantile_value) in quantile_values.iter().enumerate() { + insert_number_if_some( + &mut data_point_json, + &format!("summary_quantile_value_quantile_{}", index + 1), + &quantile_value.quantile, + ); + insert_if_some( + &mut data_point_json, + &format!("summary_quantile_value_value_{}", index + 1), + &quantile_value.value, + ); + } + } + data_points_json.push(data_point_json); + } + } + data_points_json +} + +pub fn flatten_metrics_record(metrics_record: &Metric) -> Vec> { + let mut data_points_json = Vec::new(); + let mut metric_json = BTreeMap::new(); + if let Some(gauge) = &metrics_record.gauge { + data_points_json.extend(flatten_gauge(gauge)); + } + if let Some(sum) = &metrics_record.sum { + data_points_json.extend(flatten_sum(sum)); + } + if let Some(histogram) = &metrics_record.histogram { + data_points_json.extend(flatten_histogram(histogram)); + } + if let Some(exp_histogram) = &metrics_record.exponential_histogram { + data_points_json.extend(flatten_exp_histogram(exp_histogram)); + } + if let Some(summary) = &metrics_record.summary { + data_points_json.extend(flatten_summary(summary)); + } + insert_if_some(&mut metric_json, "metric_name", &metrics_record.name); + insert_if_some( + &mut metric_json, + "metric_description", + &metrics_record.description, + ); + insert_if_some(&mut metric_json, "metric_unit", &metrics_record.unit); + insert_attributes(&mut metric_json, &metrics_record.metadata); + for data_point_json in &mut data_points_json { + for (key, value) in &metric_json { + data_point_json.insert(key.clone(), value.clone()); + } + } + data_points_json +} + +pub fn flatten_otel_metrics(body: &Bytes) -> Vec> { + let body_str = std::str::from_utf8(body).unwrap(); + let message: MetricsData = serde_json::from_str(body_str).unwrap(); + let mut vec_otel_json = Vec::new(); + if let Some(records) = &message.resource_metrics { + for record in records { + let mut resource_metrics_json = BTreeMap::new(); + if let Some(resource) = &record.resource { + insert_attributes(&mut resource_metrics_json, &resource.attributes); + insert_number_if_some( + &mut resource_metrics_json, + "resource_dropped_attributes_count", + &resource.dropped_attributes_count.map(|f| f as f64), + ); + } + let mut vec_scope_metrics_json = Vec::new(); + if let Some(scope_metrics) = &record.scope_metrics { + for scope_metric in scope_metrics { + let mut scope_metrics_json = BTreeMap::new(); + for metrics_record in &scope_metric.metrics { + vec_scope_metrics_json.extend(flatten_metrics_record(metrics_record)); + } + if let Some(scope) = &scope_metric.scope { + insert_if_some(&mut scope_metrics_json, "scope_name", &scope.name); + insert_if_some(&mut scope_metrics_json, "scope_version", &scope.version); + insert_attributes(&mut scope_metrics_json, &scope.attributes); + insert_number_if_some( + &mut scope_metrics_json, + "scope_dropped_attributes_count", + &scope.dropped_attributes_count.map(|f| f as f64), + ); + for scope_metric_json in &mut vec_scope_metrics_json { + for (key, value) in &scope_metrics_json { + scope_metric_json.insert(key.clone(), value.clone()); + } + } + } + if let Some(schema_url) = &scope_metric.schema_url { + for scope_metrics_json in &mut vec_scope_metrics_json { + scope_metrics_json.insert( + "scope_metrics_schema_url".to_string(), + Value::String(schema_url.clone()), + ); + } + } + } + } + insert_if_some( + &mut resource_metrics_json, + "resource_metrics_schema_url", + &record.schema_url, + ); + for resource_metric_json in &mut vec_scope_metrics_json { + for (key, value) in &resource_metrics_json { + resource_metric_json.insert(key.clone(), value.clone()); + } + } + vec_otel_json.extend(vec_scope_metrics_json); + } + } + vec_otel_json +} + +fn flatten_aggregation_temporality( + aggregation_temporality: &Option, +) -> BTreeMap { + let mut aggregation_temporality_json = BTreeMap::new(); + insert_number_if_some( + &mut aggregation_temporality_json, + "aggregation_temporality", + &aggregation_temporality.map(|f| f as f64), + ); + + if let Some(aggregation_temporality) = aggregation_temporality { + let description = match aggregation_temporality { + 0 => "AGGREGATION_TEMPORALITY_UNSPECIFIED", + 1 => "AGGREGATION_TEMPORALITY_DELTA", + 2 => "AGGREGATION_TEMPORALITY_CUMULATIVE", + _ => "", + }; + aggregation_temporality_json.insert( + "aggregation_temporality_description".to_string(), + Value::String(description.to_string()), + ); + } + + aggregation_temporality_json +} diff --git a/src/handlers/http/otel_traces.rs b/src/handlers/http/otel_traces.rs new file mode 100644 index 000000000..19e090059 --- /dev/null +++ b/src/handlers/http/otel_traces.rs @@ -0,0 +1,292 @@ +/* + * Parseable Server (C) 2022 - 2024 Parseable, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +use super::otel::insert_attributes; +use super::otel::insert_if_some; +use super::otel::insert_number_if_some; +use super::otel::proto::trace::v1::span::Event; +use super::otel::proto::trace::v1::span::Link; +use super::otel::proto::trace::v1::ScopeSpans; +use super::otel::proto::trace::v1::Span; +use super::otel::proto::trace::v1::Status; +use super::otel::proto::trace::v1::TracesData; +use bytes::Bytes; + +use serde_json::Value; +use std::collections::BTreeMap; + +fn flatten_scope_span(scope_span: &ScopeSpans) -> Vec> { + let mut vec_scope_span_json = Vec::new(); + let mut scope_span_json = BTreeMap::new(); + + if let Some(spans) = &scope_span.spans { + for span in spans { + let span_record_json = flatten_span_record(span); + vec_scope_span_json.extend(span_record_json); + } + } + + if let Some(scope) = &scope_span.scope { + insert_if_some(&mut scope_span_json, "scope_name", &scope.name); + insert_if_some(&mut scope_span_json, "scope_version", &scope.version); + insert_attributes(&mut scope_span_json, &scope.attributes); + insert_number_if_some( + &mut scope_span_json, + "scope_dropped_attributes_count", + &scope.dropped_attributes_count.map(|f| f as f64), + ); + + for span_json in &mut vec_scope_span_json { + for (key, value) in &scope_span_json { + span_json.insert(key.clone(), value.clone()); + } + } + } + + if let Some(schema_url) = &scope_span.schema_url { + for span_json in &mut vec_scope_span_json { + span_json.insert("schema_url".to_string(), Value::String(schema_url.clone())); + } + } + + vec_scope_span_json +} + +pub fn flatten_otel_traces(body: &Bytes) -> Vec> { + let body_str = std::str::from_utf8(body).unwrap(); + let message: TracesData = serde_json::from_str(body_str).unwrap(); + let mut vec_otel_json = Vec::new(); + + if let Some(records) = &message.resource_spans { + for record in records { + let mut resource_span_json = BTreeMap::new(); + + if let Some(resource) = &record.resource { + insert_attributes(&mut resource_span_json, &resource.attributes); + insert_number_if_some( + &mut resource_span_json, + "resource_dropped_attributes_count", + &resource.dropped_attributes_count.map(|f| f as f64), + ); + } + + let mut vec_resource_spans_json = Vec::new(); + if let Some(scope_spans) = &record.scope_spans { + for scope_span in scope_spans { + let scope_span_json = flatten_scope_span(scope_span); + vec_resource_spans_json.extend(scope_span_json); + } + } + + insert_if_some( + &mut resource_span_json, + "resource_span_schema_url", + &record.schema_url, + ); + + for resource_spans_json in &mut vec_resource_spans_json { + for (key, value) in &resource_span_json { + resource_spans_json.insert(key.clone(), value.clone()); + } + } + + vec_otel_json.extend(vec_resource_spans_json); + } + } + + vec_otel_json +} + +fn flatten_events(events: &[Event]) -> Vec> { + events + .iter() + .map(|event| { + let mut event_json = BTreeMap::new(); + insert_if_some( + &mut event_json, + "event_time_unix_nano", + &event.time_unix_nano, + ); + insert_if_some(&mut event_json, "event_name", &event.name); + insert_attributes(&mut event_json, &event.attributes); + insert_number_if_some( + &mut event_json, + "event_dropped_attributes_count", + &event.dropped_attributes_count.map(|f| f as f64), + ); + event_json + }) + .collect() +} + +fn flatten_links(links: &[Link]) -> Vec> { + links + .iter() + .map(|link| { + let mut link_json = BTreeMap::new(); + insert_if_some(&mut link_json, "link_trace_id", &link.trace_id); + insert_if_some(&mut link_json, "link_span_id", &link.span_id); + insert_attributes(&mut link_json, &link.attributes); + insert_number_if_some( + &mut link_json, + "link_dropped_attributes_count", + &link.dropped_attributes_count.map(|f| f as f64), + ); + link_json + }) + .collect() +} + +fn flatten_status(status: &Status) -> BTreeMap { + let mut status_json = BTreeMap::new(); + insert_if_some(&mut status_json, "span_status_message", &status.message); + insert_number_if_some( + &mut status_json, + "span_status_code", + &status.code.map(|f| f as f64), + ); + + if let Some(code) = status.code { + let description = match code { + 0 => "STATUS_CODE_UNSET", + 1 => "STATUS_CODE_OK", + 2 => "STATUS_CODE_ERROR", + _ => "", + }; + status_json.insert( + "span_status_description".to_string(), + Value::String(description.to_string()), + ); + } + + status_json +} + +fn flatten_flags(flags: &Option) -> BTreeMap { + let mut flags_json = BTreeMap::new(); + insert_number_if_some(&mut flags_json, "span_flags", &flags.map(|f| f as f64)); + + if let Some(flag) = flags { + let description = match flag { + 0 => "SPAN_FLAGS_DO_NOT_USE", + 255 => "SPAN_FLAGS_TRACE_FLAGS_MASK", + 256 => "SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK", + 512 => "SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK", + _ => "", + }; + flags_json.insert( + "span_flags_description".to_string(), + Value::String(description.to_string()), + ); + } + + flags_json +} + +fn flatten_kind(kind: &Option) -> BTreeMap { + let mut kind_json = BTreeMap::new(); + insert_number_if_some(&mut kind_json, "span_kind", &kind.map(|k| k as f64)); + + if let Some(kind) = kind { + let description = match kind { + 0 => "SPAN_KIND_UNSPECIFIED", + 1 => "SPAN_KIND_INTERNAL", + 2 => "SPAN_KIND_SERVER", + 3 => "SPAN_KIND_CLIENT", + 4 => "SPAN_KIND_PRODUCER", + 5 => "SPAN_KIND_CONSUMER", + _ => "", + }; + kind_json.insert( + "span_kind_description".to_string(), + Value::String(description.to_string()), + ); + } + + kind_json +} + +fn flatten_span_record(span_record: &Span) -> Vec> { + let mut span_records_json = Vec::new(); + + let mut span_record_json = BTreeMap::new(); + insert_if_some( + &mut span_record_json, + "span_trace_id", + &span_record.trace_id, + ); + insert_if_some(&mut span_record_json, "span_span_id", &span_record.span_id); + insert_if_some( + &mut span_record_json, + "span_trace_state", + &span_record.trace_state, + ); + insert_if_some( + &mut span_record_json, + "span_parent_span_id", + &span_record.parent_span_id, + ); + span_record_json.extend(flatten_flags(&span_record.flags)); + insert_if_some(&mut span_record_json, "span_name", &span_record.name); + span_record_json.extend(flatten_kind(&span_record.kind)); + insert_if_some( + &mut span_record_json, + "span_start_time_unix_nano", + &span_record.start_time_unix_nano, + ); + insert_if_some( + &mut span_record_json, + "span_end_time_unix_nano", + &span_record.end_time_unix_nano, + ); + insert_attributes(&mut span_record_json, &span_record.attributes); + insert_if_some( + &mut span_record_json, + "span_dropped_attributes_count", + &span_record.dropped_attributes_count, + ); + if let Some(events) = &span_record.events { + span_records_json.extend(flatten_events(events)); + } + insert_number_if_some( + &mut span_record_json, + "span_dropped_events_count", + &span_record.dropped_events_count.map(|f| f as f64), + ); + if let Some(links) = &span_record.links { + span_records_json.extend(flatten_links(links)); + } + + insert_number_if_some( + &mut span_record_json, + "span_dropped_links_count", + &span_record.dropped_links_count.map(|f| f as f64), + ); + + if let Some(status) = &span_record.status { + span_record_json.extend(flatten_status(status)); + } + + for span_json in &mut span_records_json { + for (key, value) in &span_record_json { + span_json.insert(key.clone(), value.clone()); + } + } + + span_records_json +} diff --git a/src/handlers/mod.rs b/src/handlers/mod.rs index e18990800..c5e7f2e6a 100644 --- a/src/handlers/mod.rs +++ b/src/handlers/mod.rs @@ -45,5 +45,17 @@ const TRINO_USER: &str = "x-trino-user"; // constants for log Source values for known sources and formats const LOG_SOURCE_KINESIS: &str = "kinesis"; +// OpenTelemetry sends logs according to the specification as explained here +// https://github.com/open-telemetry/opentelemetry-proto/tree/v1.0.0/opentelemetry/proto/logs/v1 +const LOG_SOURCE_OTEL_LOGS: &str = "otel-logs"; + +// OpenTelemetry sends traces according to the specification as explained here +// https://github.com/open-telemetry/opentelemetry-proto/tree/v1.0.0/opentelemetry/proto/metrics/v1 +const LOG_SOURCE_OTEL_METRICS: &str = "otel-metrics"; + +// OpenTelemetry sends traces according to the specification as explained here +// https://github.com/open-telemetry/opentelemetry-proto/blob/v1.0.0/opentelemetry/proto/trace/v1/trace.proto +const LOG_SOURCE_OTEL_TRACES: &str = "otel-traces"; + // AWS Kinesis constants const KINESIS_COMMON_ATTRIBUTES_KEY: &str = "x-amz-firehose-common-attributes"; diff --git a/src/kafka.rs b/src/kafka.rs index ba740df8a..deafca312 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -90,42 +90,10 @@ pub enum KafkaError { DoNotPrintError, } -// // Commented out functions -// // Might come in handy later -// fn parse_auto_env(key: &'static str) -> Result, ::Err> -// where -// T: FromStr, -// { -// Ok(if let Ok(val) = env::var(key) { -// Some(val.parse::()?) -// } else { -// None -// }) -// } - -// fn handle_duration_env_prefix(key: &'static str) -> Result, ParseIntError> { -// if let Ok(raw_secs) = env::var(format!("{key}_S")) { -// Ok(Some(Duration::from_secs(u64::from_str(&raw_secs)?))) -// } else if let Ok(raw_secs) = env::var(format!("{key}_M")) { -// Ok(Some(Duration::from_secs(u64::from_str(&raw_secs)? * 60))) -// } else { -// Ok(None) -// } -// } - -// fn parse_i32_env(key: &'static str) -> Result, KafkaError> { -// parse_auto_env::(key).map_err(|raw| KafkaError::ParseIntError(key, raw)) -// } - -// fn parse_duration_env_prefixed(key_prefix: &'static str) -> Result, KafkaError> { -// handle_duration_env_prefix(key_prefix) -// .map_err(|raw| KafkaError::ParseDurationError(key_prefix, raw)) -// } - fn setup_consumer() -> Result<(StreamConsumer, Vec), KafkaError> { if let Some(topics) = &CONFIG.parseable.kafka_topics { // topics can be a comma separated list of topics to subscribe to - let topics = topics.split(",").map(|v| v.to_owned()).collect_vec(); + let topics = topics.split(',').map(|v| v.to_owned()).collect_vec(); let host = if CONFIG.parseable.kafka_host.is_some() { CONFIG.parseable.kafka_host.as_ref() @@ -147,10 +115,6 @@ fn setup_consumer() -> Result<(StreamConsumer, Vec), KafkaError> { conf.set("client.id", val); } - // if let Some(val) = get_flag_env_val("a")? { - // conf.set("api.version.request", val.to_string()); - // } - if let Some(ssl_protocol) = CONFIG.parseable.kafka_security_protocol.as_ref() { conf.set("security.protocol", serde_json::to_string(&ssl_protocol)?); } @@ -162,8 +126,8 @@ fn setup_consumer() -> Result<(StreamConsumer, Vec), KafkaError> { // partitions is a comma separated pairs of topic:partitions let mut topic_partition_pairs = Vec::new(); let mut set = true; - for vals in vals_raw.split(",") { - let intermediate = vals.split(":").collect_vec(); + for vals in vals_raw.split(',') { + let intermediate = vals.split(':').collect_vec(); if intermediate.len() != 2 { warn!( "Value for P_KAFKA_PARTITIONS is incorrect! Skipping setting partitions!"