From dc5ca436bdc3a0b128b1d7bd446f2c55eb620a91 Mon Sep 17 00:00:00 2001 From: Preeti Dewani Date: Thu, 9 Oct 2025 17:27:34 +0530 Subject: [PATCH 1/2] add receiver, processor and exporter for kubelet and k8s_events --- deploy/pkg/k8s/monitoring.go | 96 ++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 5 deletions(-) diff --git a/deploy/pkg/k8s/monitoring.go b/deploy/pkg/k8s/monitoring.go index c18600c3..4187afad 100644 --- a/deploy/pkg/k8s/monitoring.go +++ b/deploy/pkg/k8s/monitoring.go @@ -174,6 +174,8 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "repository": pulumi.String("otel/opentelemetry-collector-contrib"), "tag": pulumi.String("0.133.0"), }, + "hostNetwork": pulumi.Bool(true), + "dnsPolicy": pulumi.String("ClusterFirstWithHostNet"), "clusterRole": pulumi.Map{ "create": pulumi.Bool(true), "rules": pulumi.Array{ @@ -183,7 +185,10 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid pulumi.String("pods"), pulumi.String("pods/log"), pulumi.String("nodes"), + pulumi.String("nodes/stats"), + pulumi.String("nodes/proxy"), pulumi.String("namespaces"), + pulumi.String("events"), }, "verbs": pulumi.StringArray{ pulumi.String("get"), @@ -191,6 +196,28 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid pulumi.String("watch"), }, }, + pulumi.Map{ + "apiGroups": pulumi.StringArray{pulumi.String("apps")}, + "resources": pulumi.StringArray{ + pulumi.String("replicasets"), + pulumi.String("deployments"), + pulumi.String("daemonsets"), + }, + "verbs": pulumi.StringArray{ + pulumi.String("get"), + pulumi.String("list"), + pulumi.String("watch"), + }, + }, + pulumi.Map{ + "nonResourceURLs": pulumi.StringArray{ + pulumi.String("/stats/*"), + pulumi.String("/metrics"), + }, + "verbs": pulumi.StringArray{ + pulumi.String("get"), + }, + }, }, }, "config": pulumi.Map{ @@ -238,9 +265,28 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid }, }, }, + "kubeletstats": pulumi.Map{ + "collection_interval": pulumi.String("60s"), + "auth_type": pulumi.String("serviceAccount"), + "endpoint": pulumi.String("https://${env:KUBERNETES_NODE_NAME}:10250"), + "insecure_skip_verify": pulumi.Bool(true), + }, + "k8s_events": pulumi.Map{ + "auth_type": pulumi.String("serviceAccount"), + "namespaces": pulumi.StringArray{ + pulumi.String("default"), + }, + }, }, "processors": pulumi.Map{ "batch": pulumi.Map{}, + "filter/kubeletstats_filter": pulumi.Map{ + "metrics": pulumi.Map{ + "datapoint": pulumi.StringArray{ + pulumi.String(`resource.attributes["k8s.namespace.name"] != "default"`), + }, + }, + }, "k8sattributes": pulumi.Map{ "auth_type": pulumi.String("serviceAccount"), "passthrough": pulumi.Bool(false), @@ -254,7 +300,6 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid pulumi.String("k8s.deployment.name"), pulumi.String("k8s.namespace.name"), pulumi.String("k8s.node.name"), - pulumi.String("k8s.pod.start_time"), pulumi.String("k8s.cluster.uid"), }, "labels": pulumi.Array{ @@ -270,11 +315,15 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "sources": pulumi.Array{ pulumi.Map{ "from": pulumi.String("resource_attribute"), - "name": pulumi.String("k8s.pod.name"), + "name": pulumi.String("k8s.pod.uid"), }, + }, + }, + pulumi.Map{ + "sources": pulumi.Array{ pulumi.Map{ "from": pulumi.String("resource_attribute"), - "name": pulumi.String("k8s.namespace.name"), + "name": pulumi.String("k8s.node.name"), }, }, }, @@ -302,14 +351,29 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "queue_size": pulumi.Int(50), }, }, + "otlphttp/victoriametrics": pulumi.Map{ + "metrics_endpoint": pulumi.String("http://victoria-metrics-victoria-metrics-single-server:8428/opentelemetry/v1/metrics"), + "timeout": pulumi.String("10s"), + "retry_on_failure": pulumi.Map{ + "enabled": pulumi.Bool(true), + "initial_interval": pulumi.String("5s"), + "max_interval": pulumi.String("30s"), + "max_elapsed_time": pulumi.String("300s"), + }, + }, }, "service": pulumi.Map{ "pipelines": pulumi.Map{ "logs": pulumi.Map{ - "receivers": pulumi.StringArray{pulumi.String("filelog")}, - "processors": pulumi.StringArray{pulumi.String("batch"), pulumi.String("k8sattributes")}, + "receivers": pulumi.StringArray{pulumi.String("filelog"), pulumi.String("k8s_events")}, + "processors": pulumi.StringArray{pulumi.String("batch")}, "exporters": pulumi.StringArray{pulumi.String("otlphttp/victorialogs")}, }, + "metrics": pulumi.Map{ + "receivers": pulumi.StringArray{pulumi.String("kubeletstats")}, + "processors": pulumi.StringArray{pulumi.String("k8sattributes"), pulumi.String("filter/kubeletstats_filter"), pulumi.String("batch")}, + "exporters": pulumi.StringArray{pulumi.String("otlphttp/victoriametrics")}, + }, }, }, }, @@ -326,6 +390,18 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "path": pulumi.String("/var/lib/docker/containers"), }, }, + pulumi.Map{ + "name": pulumi.String("proc"), + "hostPath": pulumi.Map{ + "path": pulumi.String("/proc"), + }, + }, + pulumi.Map{ + "name": pulumi.String("sys"), + "hostPath": pulumi.Map{ + "path": pulumi.String("/sys"), + }, + }, }, "extraVolumeMounts": pulumi.Array{ pulumi.Map{ @@ -338,6 +414,16 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "mountPath": pulumi.String("/var/lib/docker/containers"), "readOnly": pulumi.Bool(true), }, + pulumi.Map{ + "name": pulumi.String("proc"), + "mountPath": pulumi.String("/host/proc"), + "readOnly": pulumi.Bool(true), + }, + pulumi.Map{ + "name": pulumi.String("sys"), + "mountPath": pulumi.String("/host/sys"), + "readOnly": pulumi.Bool(true), + }, }, "extraEnvs": pulumi.Array{ pulumi.Map{ From b326abb02500f0ea35f8cbdfdcfe8fc15c29416a Mon Sep 17 00:00:00 2001 From: Preeti Dewani Date: Fri, 10 Oct 2025 01:09:10 +0530 Subject: [PATCH 2/2] add k8sattribute from pod association for log pipeline --- deploy/pkg/k8s/monitoring.go | 40 ++++++++++++------------------------ 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/deploy/pkg/k8s/monitoring.go b/deploy/pkg/k8s/monitoring.go index 4187afad..826e6ff1 100644 --- a/deploy/pkg/k8s/monitoring.go +++ b/deploy/pkg/k8s/monitoring.go @@ -232,27 +232,12 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid pulumi.Map{ "type": pulumi.String("regex_parser"), "id": pulumi.String("extract_metadata_from_filepath"), - "regex": pulumi.String(`^.*\/(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\/(?P[^\._]+)\/(?P\d+)\.log`), + "regex": pulumi.String(`^.*\/[^_]+_[^_]+_(?P[a-f0-9\-]{36})\/[^\._]+\/(?P\d+)\.log`), "parse_from": pulumi.String("attributes[\"log.file.path\"]"), "cache": pulumi.Map{ "size": pulumi.Int(128), }, }, - pulumi.Map{ - "type": pulumi.String("move"), - "from": pulumi.String("attributes.container_name"), - "to": pulumi.String("resource[\"k8s.container.name\"]"), - }, - pulumi.Map{ - "type": pulumi.String("move"), - "from": pulumi.String("attributes.namespace"), - "to": pulumi.String("resource[\"k8s.namespace.name\"]"), - }, - pulumi.Map{ - "type": pulumi.String("move"), - "from": pulumi.String("attributes.pod_name"), - "to": pulumi.String("resource[\"k8s.pod.name\"]"), - }, pulumi.Map{ "type": pulumi.String("move"), "from": pulumi.String("attributes.restart_count"), @@ -300,14 +285,8 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid pulumi.String("k8s.deployment.name"), pulumi.String("k8s.namespace.name"), pulumi.String("k8s.node.name"), - pulumi.String("k8s.cluster.uid"), - }, - "labels": pulumi.Array{ - pulumi.Map{ - "tag_name": pulumi.String("app"), - "key": pulumi.String("app"), - "from": pulumi.String("pod"), - }, + pulumi.String("container.image.name"), + pulumi.String("container.image.tag"), }, }, "pod_association": pulumi.Array{ @@ -315,7 +294,7 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "sources": pulumi.Array{ pulumi.Map{ "from": pulumi.String("resource_attribute"), - "name": pulumi.String("k8s.pod.uid"), + "name": pulumi.String("k8s.pod.ip"), }, }, }, @@ -323,7 +302,14 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "sources": pulumi.Array{ pulumi.Map{ "from": pulumi.String("resource_attribute"), - "name": pulumi.String("k8s.node.name"), + "name": pulumi.String("k8s.pod.uid"), + }, + }, + }, + pulumi.Map{ + "sources": pulumi.Array{ + pulumi.Map{ + "from": pulumi.String("connection"), }, }, }, @@ -366,7 +352,7 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid "pipelines": pulumi.Map{ "logs": pulumi.Map{ "receivers": pulumi.StringArray{pulumi.String("filelog"), pulumi.String("k8s_events")}, - "processors": pulumi.StringArray{pulumi.String("batch")}, + "processors": pulumi.StringArray{pulumi.String("k8sattributes"), pulumi.String("batch")}, "exporters": pulumi.StringArray{pulumi.String("otlphttp/victorialogs")}, }, "metrics": pulumi.Map{