[Buoyant Cloud] Replace mem.sum with mem.max (#2480)

* [Buoyant Cloud]: Replace mem.sum with mem.max The legacy `pod.container_memory_working_set_bytes.sum` metric exported by Buoyant Cloud was susceptible to double-counting during restarts, in cases where cAdvisor reports both the old and the new container simultaneously. Replace `pod.container_memory_working_set_bytes.sum` with the new `pod.container_memory_working_set_bytes.max`. Buoyant Cloud started sending this metric to Datadog on 2024.08.29. Signed-off-by: Andrew Seigner <[email protected]> * Update buoyant_cloud/metadata.csv Co-authored-by: bgoldberg122 <[email protected]> --------- Signed-off-by: Andrew Seigner <[email protected]> Co-authored-by: bgoldberg122 <[email protected]>
DataDog · Sep 11, 2024 · 2dd8153 · 2dd8153
1 parent 8727830
commit 2dd8153
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/buoyant_cloud/CHANGELOG.md b/buoyant_cloud/CHANGELOG.md
@@ -1,5 +1,11 @@
 # CHANGELOG - Buoyant Cloud
 
+## 1.2.0 / 2024-09-03
+
+***Changed***:
+
+* Replaced `pod.container_memory_working_set_bytes.sum` with `pod.container_memory_working_set_bytes.max`, to avoid double-counting memory usage during container restarts.
+
 ## 1.1.0 / 2023-05-01
 
 ***Added***:

diff --git a/buoyant_cloud/assets/dashboards/buoyant_cloud_overview.json b/buoyant_cloud/assets/dashboards/buoyant_cloud_overview.json
@@ -168,7 +168,7 @@
             "response_format": "scalar",
             "queries": [
               {
-                "query": "sum:buoyant_cloud.pod.container_memory_working_set_bytes.sum{$cluster,$namespace,$workload_name}",
+                "query": "sum:buoyant_cloud.pod.container_memory_working_set_bytes.max{$cluster,$namespace,$workload_name}",
                 "data_source": "metrics",
                 "name": "query1",
                 "aggregator": "avg"
@@ -646,7 +646,7 @@
                   "response_format": "timeseries",
                   "queries": [
                     {
-                      "query": "sum:buoyant_cloud.pod.container_memory_working_set_bytes.sum{$cluster,$namespace,$workload_name} by {cluster_name,namespace,workload_name}",
+                      "query": "sum:buoyant_cloud.pod.container_memory_working_set_bytes.max{$cluster,$namespace,$workload_name} by {cluster_name,namespace,workload_name}",
                       "data_source": "metrics",
                       "name": "query1"
                     }

diff --git a/buoyant_cloud/metadata.csv b/buoyant_cloud/metadata.csv
@@ -10,7 +10,8 @@ buoyant_cloud.link.gateway_probe_latency_ms.p99,gauge,10,millisecond,,p99 latenc
 buoyant_cloud.node.machine_cpu_cores.sum,gauge,10,core,,Node CPU cores,0,buoyant_cloud,node cpu,
 buoyant_cloud.node.machine_memory_bytes.sum,gauge,10,byte,,Node memory bytes,0,buoyant_cloud,node memory,
 buoyant_cloud.pod.container_cpu_usage_seconds.rate1m,gauge,10,core,,Container CPU cores,0,buoyant_cloud,container cpu,
-buoyant_cloud.pod.container_memory_working_set_bytes.sum,gauge,10,byte,,Container memory bytes,0,buoyant_cloud,container memory,
+buoyant_cloud.pod.container_memory_working_set_bytes.sum,gauge,10,byte,,[Deprecated] Container memory bytes,0,buoyant_cloud,container memory,
+buoyant_cloud.pod.container_memory_working_set_bytes.max,gauge,10,byte,,Container memory bytes,0,buoyant_cloud,container memory,
 buoyant_cloud.workload.inbound_http_deny.rate1m,gauge,10,response,second,HTTP responses per second denied by authorization policy,0,buoyant_cloud,inbound http deny per second,
 buoyant_cloud.workload.inbound_response.rate1m,gauge,10,response,second,HTTP responses per second,0,buoyant_cloud,inbound rps,
 buoyant_cloud.workload.inbound_response_latency_ms.p50,gauge,10,millisecond,,p50 latency,0,buoyant_cloud,inbound p50 latency,