fix(new_relic sink): Fix handling of dotted attribute names

changelog.d/21304-newrelic-log-attribute-dotted-names.fix.md

@@ -0,0 +1 @@
+Fixed a bug in the `new_relic` sinks that caused dotted attribute names to be encoded incorrectly with quotes.

src/sinks/new_relic/model.rs

@@ -8,6 +8,7 @@ use std::{
 use chrono::{DateTime, Utc};
 use ordered_float::NotNan;
 use serde::{Deserialize, Serialize};
+use vector_lib::event::ObjectMap;
 use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, UNINTENTIONAL};
 use vrl::event_path;
 
@@ -21,14 +22,13 @@ pub enum NewRelicApiModel {
     Logs(LogsApiModel),
 }
 
-type KeyValData = HashMap<KeyString, Value>;
-type DataStore = HashMap<KeyString, Vec<KeyValData>>;
+type DataStore = HashMap<KeyString, Vec<ObjectMap>>;
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct MetricsApiModel(pub Vec<DataStore>);
 
 impl MetricsApiModel {
-    pub fn new(metric_array: Vec<KeyValData>) -> Self {
+    pub fn new(metric_array: Vec<ObjectMap>) -> Self {
         let mut metric_store = DataStore::new();
         metric_store.insert("metrics".into(), metric_array);
         Self(vec![metric_store])
@@ -55,7 +55,7 @@ impl TryFrom<Vec<Event>> for MetricsApiModel {
                 // Generate Value::Object() from BTreeMap<String, String>
                 let (series, data, _) = metric.into_parts();
 
-                let mut metric_data = KeyValData::new();
+                let mut metric_data = ObjectMap::new();
 
                 // We only handle gauge and counter metrics
                 // Extract value & type and set type-related attributes
@@ -145,10 +145,10 @@ impl TryFrom<Vec<Event>> for MetricsApiModel {
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-pub struct EventsApiModel(pub Vec<KeyValData>);
+pub struct EventsApiModel(pub Vec<ObjectMap>);
 
 impl EventsApiModel {
-    pub fn new(events_array: Vec<KeyValData>) -> Self {
+    pub fn new(events_array: Vec<ObjectMap>) -> Self {
         Self(events_array)
     }
 }
@@ -160,15 +160,15 @@ impl TryFrom<Vec<Event>> for EventsApiModel {
         let mut num_non_log_events = 0;
         let mut num_nan_value = 0;
 
-        let events_array: Vec<HashMap<KeyString, Value>> = buf_events
+        let events_array: Vec<ObjectMap> = buf_events
             .into_iter()
             .filter_map(|event| {
                 let Some(log) = event.try_into_log() else {
                     num_non_log_events += 1;
                     return None;
                 };
 
-                let mut event_model = KeyValData::new();
+                let mut event_model = ObjectMap::new();
                 for (k, v) in log.convert_to_fields() {
                     event_model.insert(k, v.clone());
                 }
@@ -242,7 +242,7 @@ impl TryFrom<Vec<Event>> for EventsApiModel {
 pub struct LogsApiModel(pub Vec<DataStore>);
 
 impl LogsApiModel {
-    pub fn new(logs_array: Vec<KeyValData>) -> Self {
+    pub fn new(logs_array: Vec<ObjectMap>) -> Self {
         let mut logs_store = DataStore::new();
         logs_store.insert("logs".into(), logs_array);
         Self(vec![logs_store])
@@ -254,31 +254,47 @@ impl TryFrom<Vec<Event>> for LogsApiModel {
 
     fn try_from(buf_events: Vec<Event>) -> Result<Self, Self::Error> {
         let mut num_non_log_events = 0;
+        let mut num_non_object_events = 0;
 
-        let logs_array: Vec<HashMap<KeyString, Value>> = buf_events
+        let logs_array: Vec<ObjectMap> = buf_events
             .into_iter()
             .filter_map(|event| {
                 let Some(log) = event.try_into_log() else {
                     num_non_log_events += 1;
                     return None;
                 };
 
-                let mut log_model = KeyValData::new();
-                for (k, v) in log.convert_to_fields() {
-                    log_model.insert(k, v.clone());
-                }
-                if log.get(event_path!("message")).is_none() {
-                    log_model.insert("message".into(), Value::from("log from vector".to_owned()));
+                // We convert the log event into a logs API model simply by transmuting the type
+                // wrapper and dropping all arrays, which are not supported by the API. We could
+                // flatten out the keys, as this is what New Relic does internally, and we used to
+                // do that, but the flattening iterator accessed through
+                // `LogEvent::convert_to_fields` adds quotes to dotted fields names, which produces
+                // broken attributes in New Relic, and nesting objects is actually a (slightly) more
+                // efficient representation of the key names.
+                let (value, _metadata) = log.into_parts();
+                let Some(mut obj) = value.into_object() else {
+                    num_non_object_events += 1;
+                    return None;
+                };
+                strip_arrays(&mut obj);
+                if !obj.contains_key("message") {
+                    obj.insert("message".into(), Value::from("log from vector".to_owned()));
                 }
 
-                Some(log_model)
+                Some(obj)
             })
             .collect();
 
         if num_non_log_events > 0 {
             emit!(ComponentEventsDropped::<INTENTIONAL> {
                 count: num_non_log_events,
-                reason: "non-log event"
+                reason: "non-log event",
+            });
+        }
+        if num_non_object_events > 0 {
+            emit!(ComponentEventsDropped::<INTENTIONAL> {
+                count: num_non_object_events,
+                reason: "non-object event",
             });
         }
 
@@ -289,3 +305,12 @@ impl TryFrom<Vec<Event>> for LogsApiModel {
         }
     }
 }
+
+fn strip_arrays(obj: &mut ObjectMap) {
+    obj.retain(|_key, value| !value.is_array());
+    obj.iter_mut().for_each(|(_key, value)| {
+        if let Some(obj) = value.as_object_mut() {
+            strip_arrays(obj);
+        }
+    });
+}