[nr-k8s-otel-collector] lowdatamode/ingest cutdown (#1417)

<!--
Thank you for contributing to New Relic's Helm charts. Before you submit
this PR we'd like to
make sure you are aware of our technical requirements:

*
https://github.com/newrelic-experimental/helm-charts/blob/master/CONTRIBUTING.md#technical-requirements

For a quick overview across what we will look at reviewing your PR,
please read
our review guidelines:

*
https://github.com/newrelic-experimental/helm-charts/blob/master/REVIEW_GUIDELINES.md

Following our best practices right from the start will accelerate the
review process and
help get your PR merged quicker.

When updates to your PR are requested, please add new commits and do not
squash the
history. This will make it easier to identify new changes. The PR will
be squashed
anyways when it is merged. Thanks.

For fast feedback, please @-mention maintainers that are listed in the
Chart.yaml file.

Please make sure you test your changes before you push them. Once
pushed, a Github Action
will run across your changes and do some initial checks and linting.
These checks run
very quickly. Please check the results. We would like these checks to
pass before we
even continue reviewing your changes.
-->
#### Is this a new chart

#### What this PR does / why we need it:

#### Which issue this PR fixes
*(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)`
format, will close that issue when PR gets merged)*
  - fixes #

#### Special notes for your reviewer:

#### Checklist
[Place an '[x]' (no spaces) in all applicable fields. Please remove
unrelated fields.]
- [ ] Chart Version bumped
- [ ] Variables are documented in the README.md
- [ ] Title of the PR starts with chart name (e.g. `[mychartname]`)

---------

Co-authored-by: Daniel Budziwojski <[email protected]>
Co-authored-by: Daniel Budziwojski <[email protected]>

charts/nr-k8s-otel-collector/Chart.yaml

@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.3
+version: 0.6.0
 
 dependencies:
   - name: common-library

charts/nr-k8s-otel-collector/README.md

@@ -105,6 +105,7 @@ daemonset:
 | kube-state-metrics.prometheusScrape | bool | `false` | Disable prometheus from auto-discovering KSM and potentially scraping duplicated data |
 | labels | object | `{}` | Additional labels for chart objects |
 | licenseKey | string | `""` | This set this license key to use. Can be configured also with `global.licenseKey` |
+| lowDataMode | bool | `false` | Send only the metrics required to light up the NR kubernetes UI |
 | nodeSelector | object | `{}` | Sets all pods' node selector. Can be configured also with `global.nodeSelector` |
 | nrStaging | bool | `false` | Send the metrics to the staging backend. Requires a valid staging license key. Can be configured also with `global.nrStaging` |
 | podLabels | object | `{}` | Additional labels for chart pods |
@@ -114,14 +115,12 @@ daemonset:
 | rbac.create | bool | `true` | Specifies whether RBAC resources should be created |
 | receivers.filelog.enabled | bool | `true` | Specifies whether the `filelog` receiver is enabled |
 | receivers.hostmetrics.enabled | bool | `true` | Specifies whether the `hostmetrics` receiver is enabled |
-| receivers.hostmetrics.scrapeInterval | string | `20s` | Sets the scrape interval for the `hostmetrics` receiver |
-| receivers.k8sCluster.enabled | bool | `true` | Specifies whether the `k8s_cluster` receiver is enabled |
-| receivers.k8sCluster.scrapeInterval | string | `20s` | Sets the scrape interval for the `k8s_cluster` receiver |
+| receivers.hostmetrics.scrapeInterval | string | `1m` | Sets the scrape interval for the `hostmetrics` receiver |
 | receivers.k8sEvents.enabled | bool | `true` | Specifies whether the `k8s_events` receiver is enabled |
 | receivers.kubeletstats.enabled | bool | `true` | Specifies whether the `kubeletstats` receiver is enabled |
-| receivers.kubeletstats.scrapeInterval | string | `20s` | Sets the scrape interval for the `kubeletstats` receiver |
+| receivers.kubeletstats.scrapeInterval | string | `1m` | Sets the scrape interval for the `kubeletstats` receiver |
 | receivers.prometheus.enabled | bool | `true` | Specifies whether the `prometheus` receiver is enabled |
-| receivers.prometheus.scrapeInterval | string | `20s` | Sets the scrape interval for the `prometheus` receiver |
+| receivers.prometheus.scrapeInterval | string | `1m` | Sets the scrape interval for the `prometheus` receiver |
 | serviceAccount | object | See `values.yaml` | Settings controlling ServiceAccount creation |
 | serviceAccount.create | bool | `true` | Specifies whether a ServiceAccount should be created |
 | tolerations | list | `[]` | Sets all pods' tolerations to node taints. Can be configured also with `global.tolerations` |

charts/nr-k8s-otel-collector/templates/daemonset-configmap.yaml

@@ -205,6 +205,111 @@ data:
             to: resource["k8s.pod.uid"]
 
     processors:
+      metricstransform/ldm:
+        transforms:
+          - include: .*
+            match_type: regexp
+            action: update
+            operations:
+            - action: add_label
+              new_label: low.data.mode
+              new_value: 'false'
+
+      metricstransform/kubeletstats:
+        transforms:
+          - include: container\.(cpu\.utilization|filesystem\.(capacity|usage)|memory\.usage)
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+          - include: k8s\.node\.(cpu\.(time|utilization)|filesystem\.(capacity|usage)|memory\.(available|working_set))
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+          - include: k8s\.pod\.(filesystem\.(available|capacity|usage)|memory\.(working_set))
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+
+      metricstransform/cadvisor:
+        transforms:
+          - include: container_cpu_(cfs_(periods_total|throttled_periods_total)|usage_seconds_total)
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+          - include: container_memory_working_set_bytes
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+          - include: container_network_(working_set_bytes|receive_(bytes_total|errors_total)|transmit_(bytes_total|errors_total))
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+          - include: container_spec_memory_limit_bytes
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+
+      metricstransform/kubelet:
+        transforms:
+          - include: go_(goroutines|threads)
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+          - include: process_resident_memory_bytes
+            action: update
+            match_type: regexp
+            operations:
+            - action: update_label
+              label: low.data.mode
+              value_actions:
+              - value: 'false'
+                new_value: 'true'
+
+      filter/exclude_metrics_low_data_mode:
+        metrics:
+          metric:
+            - 'HasAttrOnDatapoint("low.data.mode", "false")'
+
       transform/truncate:
         log_statements:
           - context: log
@@ -295,6 +400,22 @@ data:
           - key: service_name
             action: delete
 
+      transform/low_data_mode_inator:
+        metric_statements:
+          - context: metric
+            statements:
+              - set(description, "")
+              - set(unit, "")
+
+      resource/low_data_mode_inator:
+        attributes:
+          - key: http.scheme
+            action: delete
+          - key: net.host.name
+            action: delete
+          - key: net.host.port
+            action: delete
+
       k8sattributes:
         auth_type: "serviceAccount"
         passthrough: false
@@ -328,7 +449,15 @@ data:
             action: upsert
             from_attribute: namespace
 
+      memory_limiter:
+         check_interval: 1s
+         limit_percentage: 80
+         spike_limit_percentage: 25
+
       batch:
+        send_batch_max_size: 1000
+        timeout: 30s
+        send_batch_size : 800
 
     exporters:
       otlphttp/newrelic:
@@ -351,7 +480,7 @@ data:
         {{- if or .Values.receivers.hostmetrics.enabled (or .Values.receivers.kubeletstats.enabled .Values.receivers.prometheus.enabled) }}
         metrics:
           receivers:
-            {{- if and .Values.receivers.hostmetrics.enabled (include "newrelic.common.privileged" .) }}
+            {{- if and .Values.receivers.hostmetrics.enabled (include "newrelic.common.privileged" .) (not (include "newrelic.common.lowDataMode" .)) }}
             - hostmetrics
             {{- end }}
             {{- if .Values.receivers.kubeletstats.enabled }}
@@ -361,7 +490,14 @@ data:
             - prometheus
             {{- end }}
           processors:
-            # - transform/truncate
+            {{- if include "newrelic.common.lowDataMode" . }}
+            - metricstransform/ldm
+            - metricstransform/kubeletstats
+            - metricstransform/cadvisor
+            - metricstransform/kubelet
+            - filter/exclude_metrics_low_data_mode
+            {{- end }}
+            - transform/truncate
             - filter/exclude_cpu_utilization
             - filter/exclude_memory_utilization
             - filter/exclude_memory_usage
@@ -374,8 +510,13 @@ data:
             - attributes/exclude_system_paging
             - resourcedetection/cloudproviders
             - resource
+            {{- if include "newrelic.common.lowDataMode" . }}
+            - transform/low_data_mode_inator
+            - resource/low_data_mode_inator
+            {{- end }}
             - k8sattributes
             - attributes/self
+            - memory_limiter
             - batch
           exporters:
             - otlphttp/newrelic
@@ -384,7 +525,11 @@ data:
         logs:
           receivers:
             - filelog
-          processors: [transform/truncate, resource, k8sattributes, batch]
+          processors:
+            - transform/truncate
+            - resource
+            - k8sattributes
+            - batch
           exporters:
             - otlphttp/newrelic
         {{- end }}