Merge branch 'main' into alew/ob-35960

observeinc · Aug 29, 2024 · f0d464a · f0d464a
2 parents 4918307 + eaf1265
commit f0d464a
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 44 deletions.
diff --git a/charts/agent/Chart.lock b/charts/agent/Chart.lock
@@ -11,5 +11,5 @@ dependencies:
 - name: opentelemetry-collector
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
   version: 0.101.1
-digest: sha256:987a47d694c8ea67e5dbcfa46bbdf84702bd9c8dee497fda7b8ff4c6f2e90a43
-generated: "2024-08-27T13:58:03.699484-07:00"
+digest: sha256:a24a099b6ef2b3229aa5436611180f6238d22e1d5e55edc2d14465b55bc28392
+generated: "2024-08-28T20:50:31.245596259Z"
diff --git a/charts/agent/Chart.yaml b/charts/agent/Chart.yaml
@@ -2,25 +2,30 @@ apiVersion: v2
 name: agent
 description: Chart to install K8s collection stack based on Observe Agent
 type: application
-version: 0.6.1
+version: 0.7.0
 appVersion: "1.0.0"
 dependencies:
   - name: opentelemetry-collector
     version: 0.101.1
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
     alias: deployment-cluster-events
+    condition: cluster.events.enabled
   - name: opentelemetry-collector
     version: 0.101.1
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
     alias: deployment-cluster-metrics
+    condition: cluster.metrics.enabled
   - name: opentelemetry-collector
     version: 0.101.1
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
     alias: daemonset-logs-metrics
+    # TODO change to tags
+    condition: containers.logs.enabled
   - name: opentelemetry-collector
     version: 0.101.1
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
     alias: deployment-agent-monitor
+    condition: agent.selfMonitor.enabled
 maintainers:
   - name: Observe
     email: [email protected]
diff --git a/charts/agent/README.md b/charts/agent/README.md
@@ -1,6 +1,6 @@
 # agent
 
-![Version: 0.6.1](https://img.shields.io/badge/Version-0.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
+![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
 
 > [!CAUTION]
 > This chart is under active development and is not meant to be installed yet.
@@ -26,13 +26,19 @@ Chart to install K8s collection stack based on Observe Agent
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| agent.config.global.debug.verbosity | string | `"basic"` |  |
+| agent.config.global.processors.batch.send_batch_max_size | int | `100` |  |
+| agent.config.global.processors.batch.send_batch_size | int | `100` |  |
+| agent.config.global.service.telemetry.logging_level | string | `"WARN"` |  |
+| agent.config.global.service.telemetry.metrics_level | string | `"normal"` |  |
+| agent.selfMonitor.enabled | bool | `true` |  |
+| cluster.events.enabled | bool | `true` |  |
 | cluster.events.pullInterval | string | `"20m"` |  |
+| cluster.metrics.enabled | bool | `true` |  |
 | cluster.name | string | `"observe-agent-monitored-cluster"` |  |
-| config.global.debug.verbosity | string | `"basic"` |  |
-| config.global.processors.batch.send_batch_max_size | int | `100` |  |
-| config.global.processors.batch.send_batch_size | int | `100` |  |
-| config.global.service.telemetry.logging_level | string | `"WARN"` |  |
-| config.global.service.telemetry.metrics_level | string | `"normal"` |  |
+| cluster.namespaceOverride.value | string | `"observe"` |  |
+| containers.logs.enabled | bool | `true` |  |
+| containers.metrics.enabled | bool | `true` |  |
 | daemonset-logs-metrics.clusterRole.create | bool | `false` |  |
 | daemonset-logs-metrics.clusterRole.name | string | `"observe-agent-cluster-role"` |  |
 | daemonset-logs-metrics.command.extraArgs[0] | string | `"start"` |  |
@@ -41,6 +47,7 @@ Chart to install K8s collection stack based on Observe Agent
 | daemonset-logs-metrics.command.name | string | `"observe-agent"` |  |
 | daemonset-logs-metrics.configMap.create | bool | `false` |  |
 | daemonset-logs-metrics.configMap.existingName | string | `"daemonset-logs-metrics"` |  |
+| daemonset-logs-metrics.enabled | bool | `true` |  |
 | daemonset-logs-metrics.extraEnvsFrom | list | `[]` |  |
 | daemonset-logs-metrics.extraEnvs[0].name | string | `"OBSERVE_CLUSTER_NAME"` |  |
 | daemonset-logs-metrics.extraEnvs[0].valueFrom.configMapKeyRef.key | string | `"name"` |  |
@@ -123,6 +130,7 @@ Chart to install K8s collection stack based on Observe Agent
 | deployment-agent-monitor.command.name | string | `"observe-agent"` |  |
 | deployment-agent-monitor.configMap.create | bool | `false` |  |
 | deployment-agent-monitor.configMap.existingName | string | `"deployment-agent-monitor"` |  |
+| deployment-agent-monitor.enabled | bool | `true` |  |
 | deployment-agent-monitor.extraEnvsFrom | list | `[]` |  |
 | deployment-agent-monitor.extraEnvs[0].name | string | `"OBSERVE_CLUSTER_NAME"` |  |
 | deployment-agent-monitor.extraEnvs[0].valueFrom.configMapKeyRef.key | string | `"name"` |  |
@@ -237,6 +245,7 @@ Chart to install K8s collection stack based on Observe Agent
 | deployment-cluster-metrics.command.name | string | `"observe-agent"` |  |
 | deployment-cluster-metrics.configMap.create | bool | `false` |  |
 | deployment-cluster-metrics.configMap.existingName | string | `"deployment-cluster-metrics"` |  |
+| deployment-cluster-metrics.enabled | bool | `true` |  |
 | deployment-cluster-metrics.extraEnvsFrom | list | `[]` |  |
 | deployment-cluster-metrics.extraEnvs[0].name | string | `"OBSERVE_CLUSTER_NAME"` |  |
 | deployment-cluster-metrics.extraEnvs[0].valueFrom.configMapKeyRef.key | string | `"name"` |  |
@@ -286,7 +295,6 @@ Chart to install K8s collection stack based on Observe Agent
 | deployment-cluster-metrics.resources | object | `{"requests":{"cpu":"250m","memory":"256Mi"}}` | --------------------------------------- # Same for each deployment/daemonset      # |
 | deployment-cluster-metrics.serviceAccount.create | bool | `false` |  |
 | deployment-cluster-metrics.serviceAccount.name | string | `"observe-agent-service-account"` |  |
-| namespaceOverride | string | `"observe"` |  |
 | observe.collectionEndpoint.value | string | `""` |  |
 | observe.entityToken.create | bool | `false` |  |
 | observe.entityToken.value | string | `""` |  |

diff --git a/charts/agent/templates/_config-exporters.tpl b/charts/agent/templates/_config-exporters.tpl
@@ -23,7 +23,7 @@ prometheusremotewrite:
 
 {{- define "config.exporters.debug" -}}
 debug/override:
-    verbosity: {{ .Values.config.global.debug.verbosity }}
+    verbosity: {{ .Values.agent.config.global.debug.verbosity }}
     sampling_initial: 2
     sampling_thereafter: 1
 {{- end -}}
diff --git a/charts/agent/templates/_config-processors.tpl b/charts/agent/templates/_config-processors.tpl
@@ -7,8 +7,8 @@ resourcedetection/cloud:
 
 {{- define "config.processors.batch" -}}
 batch:
-  send_batch_size: {{ .Values.config.global.processors.batch.send_batch_size }}
-  send_batch_max_size: {{ .Values.config.global.processors.batch.send_batch_max_size }}
+  send_batch_size: {{ .Values.agent.config.global.processors.batch.send_batch_size }}
+  send_batch_max_size: {{ .Values.agent.config.global.processors.batch.send_batch_max_size }}
 {{- end -}}
 
 {{- define "config.processors.attributes.k8sattributes" -}}

diff --git a/charts/agent/templates/_config-telemetry.tpl b/charts/agent/templates/_config-telemetry.tpl
@@ -1,8 +1,8 @@
 {{- define "config.service.telemetry" -}}
 telemetry:
     metrics:
-      level: {{ .Values.config.global.service.telemetry.metrics_level }}
+      level: {{ .Values.agent.config.global.service.telemetry.metrics_level }}
       address: {{ template "config.local_host"}}:8888
     logs:
-      level: {{ .Values.config.global.service.telemetry.logging_level }}
+      level: {{ .Values.agent.config.global.service.telemetry.logging_level }}
 {{- end -}}
diff --git a/charts/agent/templates/_deployment-cluster-events-config.tpl b/charts/agent/templates/_deployment-cluster-events-config.tpl
@@ -123,7 +123,7 @@ processors:
           - set(attributes["observe_transform"]["facets"]["deploymentName"], body["metadata"]["ownerReferences"][0]["name"]) where body["metadata"]["ownerReferences"][0]["kind"] == "ReplicaSet"
           - replace_pattern(attributes["observe_transform"]["facets"]["deploymentName"], "^(.*)-[0-9a-f]+$$", "$$1")
           - set(attributes["observe_transform"]["facets"]["deploymentName"], body["metadata"]["ownerReferences"][0]["name"]) where body["metadata"]["ownerReferences"][0]["kind"] == "Deployment"
-      # For Pod
+      # Pod
       - context: log
         conditions:
           - body["kind"] == "Pod"
@@ -135,13 +135,13 @@ processors:
           - set(attributes["observe_transform"]["facets"]["startTime"], body["status"]["startTime"])
           - set(attributes["observe_transform"]["facets"]["readinessGates"], body["object"]["spec"]["readinessGates"])
           - set(attributes["observe_transform"]["facets"]["nodeName"], body["spec"]["nodeName"])
-      # For Namespace
+      # Namespace
       - context: log
         conditions:
           - body["kind"] == "Namespace"
         statements:
           - set(attributes["observe_transform"]["facets"]["status"], body["status"]["phase"])
-      # For Node
+      # Node
       - context: log
         conditions:
           - body["kind"] == "Node"
@@ -151,7 +151,7 @@ processors:
           - set(attributes["observe_transform"]["facets"]["kubeletVersion"], body["status"]["nodeInfo"]["kubeletVersion"])
           - set(attributes["observe_transform"]["facets"]["osImage"], body["status"]["nodeInfo"]["osImage"])
           - set(attributes["observe_transform"]["facets"]["taints"], body["spec"]["taints"])
-      # For Deployment
+      # Deployment
       - context: log
         conditions:
           - body["kind"] == "Deployment"
@@ -163,7 +163,7 @@ processors:
           - set(attributes["observe_transform"]["facets"]["readyReplicas"], body["status"]["readyReplicas"])
           - set(attributes["observe_transform"]["facets"]["readyReplicas"], 0) where attributes["observe_transform"]["facets"]["readyReplicas"] == nil
           - set(attributes["observe_transform"]["facets"]["unavailableReplicas"], body["status"]["unavailableReplicas"])
-      # For ReplicaSet
+      # ReplicaSet
       - context: log
         conditions:
           - body["kind"] == "ReplicaSet"
@@ -173,7 +173,7 @@ processors:
           - set(attributes["observe_transform"]["facets"]["availableReplicas"], body["status"]["availableReplicas"])
           - set(attributes["observe_transform"]["facets"]["readyReplicas"], body["status"]["readyReplicas"])
           - set(attributes["observe_transform"]["facets"]["readyReplicas"], 0) where attributes["observe_transform"]["facets"]["readyReplicas"] == nil
-      # For Event
+      # Event
       - context: log
         conditions:
           - body["kind"] == "Event"
@@ -186,6 +186,64 @@ processors:
           - set(attributes["observe_transform"]["facets"]["count"], body["count"])
           - set(attributes["observe_transform"]["facets"]["type"], body["type"])
           - set(attributes["observe_transform"]["facets"]["sourceComponent"], body["source"]["component"])
+      # Job
+      - context: log
+        conditions:
+          - body["kind"] == "Job"
+        statements:
+          # status
+          - set(attributes["observe_transform"]["facets"]["startTime"], body["status"]["startTime"])
+          - set(attributes["observe_transform"]["facets"]["activePods"], body["status"]["active"])
+          - set(attributes["observe_transform"]["facets"]["failedPods"], body["status"]["falied"])
+          - set(attributes["observe_transform"]["facets"]["succeededPods"], body["status"]["succeeded"])
+          - set(attributes["observe_transform"]["facets"]["readyPods"], body["status"]["ready"])
+          # spec
+          - set(attributes["observe_transform"]["facets"]["completions"], body["spec"]["completions"])
+          - set(attributes["observe_transform"]["facets"]["parallelism"], body["spec"]["parallelism"])
+          - set(attributes["observe_transform"]["facets"]["activeDeadlineSeconds"], body["spec"]["activeDeadlineSeconds"])
+          - set(attributes["observe_transform"]["facets"]["backoffLimit"], body["spec"]["backoffLimit"])
+      # CronJob
+      - context: log
+        conditions:
+          - body["kind"] == "CronJob"
+        statements:
+          # spec
+          - set(attributes["observe_transform"]["facets"]["schedule"], body["spec"]["schedule"])
+          - set(attributes["observe_transform"]["facets"]["suspend"], "Active") where body["spec"]["suspend"] == false
+          - set(attributes["observe_transform"]["facets"]["suspend"], "Suspend") where body["spec"]["suspend"] == true
+          # status
+          - set(attributes["observe_transform"]["facets"]["activeJobs"], Len(body["status"]["active"]))
+      # DaemonSet
+      - context: log
+        conditions:
+          - body["kind"] == "DaemonSet"
+        statements:
+          # status
+          - set(attributes["observe_transform"]["facets"]["numberReady"], body["status"]["numberReady"])
+          - set(attributes["observe_transform"]["facets"]["desiredNumberScheduled"], body["status"]["desiredNumberScheduled"])
+          - set(attributes["observe_transform"]["facets"]["currentNumberScheduled"], body["status"]["currentNumberScheduled"])
+          - set(attributes["observe_transform"]["facets"]["updatedNumberScheduled"], body["status"]["updatedNumberScheduled"])
+          - set(attributes["observe_transform"]["facets"]["numberAvailable"], body["status"]["numberAvailable"])
+          - set(attributes["observe_transform"]["facets"]["numberUnavailable"], body["status"]["numberUnavailable"])
+          - set(attributes["observe_transform"]["facets"]["numberMisscheduled"], body["status"]["numberMisscheduled"])
+          # spec
+          - set(attributes["observe_transform"]["facets"]["updateStrategy"], body["spec"]["updateStrategy"]["type"])
+          - set(attributes["observe_transform"]["facets"]["maxUnavailable"], body["spec"]["updateStrategy"]["rollingUpdate"]["maxUnavailable"])
+          - set(attributes["observe_transform"]["facets"]["maxSurge"], body["spec"]["updateStrategy"]["rollingUpdate"]["maxSurge"])
+      # StatefulSet
+      - context: log
+        conditions:
+          - body["kind"] == "StatefulSet"
+        statements:
+          # status
+          - set(attributes["observe_transform"]["facets"]["currentReplicas"], body["status"]["currentReplicas"])
+          - set(attributes["observe_transform"]["facets"]["readyReplicas"], body["status"]["readyReplicas"])
+          # spec
+          - set(attributes["observe_transform"]["facets"]["service"], body["spec"]["serviceName"])
+          - set(attributes["observe_transform"]["facets"]["podManagementPolicy"], body["spec"]["podManagementPolicy"])
+          - set(attributes["observe_transform"]["facets"]["desiredReplicas"], body["spec"]["replicas"])
+          - set(attributes["observe_transform"]["facets"]["updateStrategy"], body["spec"]["updateStrategy"]["type"])
+          - set(attributes["observe_transform"]["facets"]["partition"], body["spec"]["updateStrategy"]["rollingUpdate"]["partition"])
 
   # drop all namespace except kube-system
   filter/cluster:

diff --git a/charts/agent/templates/_helpers.tpl b/charts/agent/templates/_helpers.tpl
@@ -1,6 +1,6 @@
 {{- define "observe-agent.namespace" -}}
-  {{- if .Values.namespaceOverride -}}
-    {{- .Values.namespaceOverride -}}
+  {{- if .Values.cluster.namespaceOverride.value -}}
+    {{- .Values.cluster.namespaceOverride.value -}}
   {{- else -}}
     "observe"
   {{- end -}}

diff --git a/charts/agent/testit.yaml b/charts/agent/testit.yaml
diff --git a/charts/agent/values.yaml b/charts/agent/values.yaml
@@ -1,8 +1,5 @@
-
-namespaceOverride: &namespaceOverride "observe"
-
 observe:
-  #If create = false it is assumed that a secret named agent-credentials with key: OBSERVE_TOKEN already exists
+  # If create = false it is assumed that a secret named agent-credentials with key: OBSERVE_TOKEN already exists
   token:
     create: true
     value: ""
@@ -18,30 +15,47 @@ cluster:
   name: observe-agent-monitored-cluster
   events:
     pullInterval: 20m
+    enabled: true
+  metrics:
+    enabled: true
+
+  namespaceOverride:
+    # ! This needs to have same value as namespaceOverride in deployments and daemonsets below
+    value: observe
 
-config:
-  global:
-    processors:
-      batch:
-        send_batch_size: 100
-        send_batch_max_size: 100
-    service:
-      telemetry:
-        metrics_level: normal
-        logging_level: WARN
-    debug:
-      # values basic, normal, detailed
-      verbosity: basic
+containers:
+  logs:
+    enabled: true
+  metrics:
+    enabled: true
+
+agent:
+  config:
+    global:
+      processors:
+        batch:
+          send_batch_size: 100
+          send_batch_max_size: 100
+      service:
+        telemetry:
+          metrics_level: normal
+          logging_level: WARN
+      debug:
+        # values basic, normal, detailed
+        verbosity: basic
+  selfMonitor:
+    enabled: true
 
 
 ################################################
 deployment-cluster-events:
+  # enabled: true
   mode: deployment
 
   # ----------------------------------------- #
   # Different for each deployment/daemonset #
   nameOverride: "deployment-cluster-events"
-  namespaceOverride: *namespaceOverride
+  namespaceOverride: "observe"
 
   configMap:
     create: false
@@ -150,11 +164,12 @@ deployment-cluster-events:
 
 ################################################
 deployment-cluster-metrics:
+  enabled: true
   mode: deployment
   # ----------------------------------------- #
   # Different for each deployment/daemonset #
   nameOverride: "deployment-cluster-metrics"
-  namespaceOverride: *namespaceOverride
+  namespaceOverride: "observe"
 
   configMap:
     create: false
@@ -262,11 +277,12 @@ deployment-cluster-metrics:
 
 ################################################
 daemonset-logs-metrics:
+  enabled: true
   mode: daemonset
   # ----------------------------------------- #
   # Different for each deployment/daemonset #
   nameOverride: "daemonset-logs-metrics"
-  namespaceOverride: *namespaceOverride
+  namespaceOverride: "observe"
 
   configMap:
     create: false
@@ -406,11 +422,12 @@ daemonset-logs-metrics:
 
 ################################################
 deployment-agent-monitor:
+  enabled: true
   mode: deployment
   # ----------------------------------------- #
   # Different for each deployment/daemonset #
   nameOverride: "deployment-agent-monitor"
-  namespaceOverride: *namespaceOverride
+  namespaceOverride: "observe"
 
   configMap:
     create: false