From f93eabef0a177ffb8788a9bce894aa6bc152683c Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Fri, 26 Jul 2024 10:19:48 +0200 Subject: [PATCH] feat: add ECS mode for Helm deployment (#33) * remove unused daemonset.yaml file * feat: add elasticsearch k8s secrets * docs: add elasticsearch endpoint secrets * set daemonset as default Helm deployment method * Revert "set daemonset as default Helm deployment method" This reverts commit 80612a5a2ceb022b50975b68ab035eda9b8c04c3. * Revert "remove unused daemonset.yaml file" This reverts commit 8de40036a024fb280e8d8201329a0ed46f997acb. * decouple elasticsearch exporter from deployment mode * feat: add configmap for daemonset deployment * rollback filelog pods exclusion * feat: add deployment.environment attribute * remove extra volumes in deployment mode * remove security context from deployment mode * add security context for daemonset deployment --- .env.override | 2 +- .github/README.md | 27 +- .../elastic-helm/configmap-daemonset.yaml | 265 ++++++++++++++++++ ...elastic.yaml => configmap-deployment.yaml} | 2 + kubernetes/elastic-helm/daemonset.yaml | 71 +++-- .../{values.yaml => deployment.yaml} | 0 6 files changed, 322 insertions(+), 45 deletions(-) create mode 100644 kubernetes/elastic-helm/configmap-daemonset.yaml rename kubernetes/elastic-helm/{configmap-elastic.yaml => configmap-deployment.yaml} (97%) rename kubernetes/elastic-helm/{values.yaml => deployment.yaml} (100%) diff --git a/.env.override b/.env.override index ad0661922a..53ffb483f9 100644 --- a/.env.override +++ b/.env.override @@ -14,6 +14,6 @@ FRAUD_SERVICE_DOCKERFILE=./src/frauddetectionservice/Dockerfile.elastic # ********************* # Elastic Collector # ********************* -COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-7b611e39-SNAPSHOT +COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-SNAPSHOT OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml diff --git a/.github/README.md b/.github/README.md index 4958b2bd73..8136108a34 100644 --- a/.github/README.md +++ b/.github/README.md @@ -49,10 +49,10 @@ Additionally, the OpenTelemetry Contrib collector has also been changed to the [ helm repo update open-telemetry # deploy the configuration for the Elastic OpenTelemetry collector distribution - kubectl apply -f configmap-elastic.yaml + kubectl apply -f configmap-deployment.yaml # deploy the demo through helm install - helm install -f values.yaml my-otel-demo open-telemetry/opentelemetry-demo + helm install -f deployment.yaml my-otel-demo open-telemetry/opentelemetry-demo ``` #### Kubernetes monitoring @@ -60,10 +60,27 @@ Additionally, the OpenTelemetry Contrib collector has also been changed to the [ This demo already enables cluster level metrics collection with `clusterMetrics` and Kubernetes events collection with `kubernetesEvents`. -In order to add Node level metrics collection and autodiscovery for Redis Pods -we can run an additional Otel collector Daemonset with the following: +In order to add Node level metrics collection we can run an additional Otel collector Daemonset with the following: -`helm install daemonset open-telemetry/opentelemetry-collector --values daemonset.yaml` +1. Create a secret in Kubernetes with the following command. + ``` + kubectl create secret generic elastic-secret-ds \ + --from-literal=elastic_endpoint='YOUR_ELASTICSEARCH_ENDPOINT' \ + --from-literal=elastic_api_key='YOUR_ELASTICSEARCH_API_KEY' + ``` + Don't forget to replace + - `YOUR_ELASTICSEARCH_ENDPOINT`: your Elasticsearch endpoint (example: `1234567.us-west2.gcp.elastic-cloud.com:443`). + - `YOUR_ELASTICSEARCH_API_KEY`: your Elasticsearch API Key + +2. Execute the following command to deploy the OpenTelemetry Collector to your Kubernetes cluster: + +``` +# deploy the configuration for the Elastic OpenTelemetry collector distribution +kubectl apply -f configmap-daemonset.yaml + +# deploy the Elastic OpenTelemetry collector distribution through helm install +helm install otel-daemonset open-telemetry/opentelemetry-collector --values daemonset.yaml +``` ## Explore and analyze the data With Elastic diff --git a/kubernetes/elastic-helm/configmap-daemonset.yaml b/kubernetes/elastic-helm/configmap-daemonset.yaml new file mode 100644 index 0000000000..6385c19328 --- /dev/null +++ b/kubernetes/elastic-helm/configmap-daemonset.yaml @@ -0,0 +1,265 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-otelcol-agent-ds + namespace: default + labels: + app.kubernetes.io/name: otelcol + +data: + relay: | + exporters: + debug: + elasticsearch: + endpoints: + - ${env:ELASTIC_ENDPOINT} + api_key: ${env:ELASTIC_API_KEY} + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + mapping: + mode: ecs + processors: + batch: {} + elasticinframetrics: + add_system_metrics: true + add_k8s_metrics: true + resourcedetection/eks: + detectors: [env, eks] + timeout: 15s + override: true + eks: + resource_attributes: + k8s.cluster.name: + enabled: true + resourcedetection/gcp: + detectors: [env, gcp] + timeout: 2s + override: false + resource/k8s: + attributes: + - key: service.name + from_attribute: app.label.component + action: insert + attributes/k8s_logs_dataset: + actions: + - key: data_stream.dataset + value: "kubernetes.container_logs" + action: upsert + attributes/dataset: + actions: + - key: event.dataset + from_attribute: data_stream.dataset + action: upsert + resource/cloud: + attributes: + - key: cloud.instance.id + from_attribute: host.id + action: insert + resource/demo: + attributes: + - key: deployment.environment + value: "opentelemetry-demo" + action: upsert + resource/process: + attributes: + - key: process.executable.name + action: delete + - key: process.executable.path + action: delete + resourcedetection/system: + detectors: ["system", "ec2"] + system: + hostname_sources: [ "os" ] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true + ec2: + resource_attributes: + host.name: + enabled: false + host.id: + enabled: true + k8sattributes: + filter: + node_from_env_var: K8S_NODE_NAME + passthrough: false + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: connection + extract: + metadata: + - "k8s.namespace.name" + - "k8s.deployment.name" + - "k8s.statefulset.name" + - "k8s.daemonset.name" + - "k8s.cronjob.name" + - "k8s.job.name" + - "k8s.node.name" + - "k8s.pod.name" + - "k8s.pod.uid" + - "k8s.pod.start_time" + labels: + - tag_name: app.label.component + key: app.kubernetes.io/component + from: pod + receivers: + filelog: + retry_on_failure: + enabled: true + start_at: end + exclude: + # exlude collector logs + - /var/log/pods/default_otel-daemonset-opentelemetry-collector-agent*_*/opentelemetry-collector/*.log + include: + - /var/log/pods/*/*/*.log + include_file_name: false + include_file_path: true + operators: + - id: container-parser + type: container + hostmetrics: + collection_interval: 10s + root_path: /hostfs + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true + system.cpu.logical.count: + enabled: true + memory: + metrics: + system.memory.utilization: + enabled: true + process: + mute_process_exe_error: true + mute_process_io_error: true + mute_process_user_error: true + metrics: + process.threads: + enabled: true + process.open_file_descriptors: + enabled: true + process.memory.utilization: + enabled: true + process.disk.operations: + enabled: true + network: + processes: + load: + disk: + filesystem: + exclude_mount_points: + mount_points: + - /dev/* + - /proc/* + - /sys/* + - /run/k3s/containerd/* + - /var/lib/docker/* + - /var/lib/kubelet/* + - /snap/* + match_type: regexp + exclude_fs_types: + fs_types: + - autofs + - binfmt_misc + - bpf + - cgroup2 + - configfs + - debugfs + - devpts + - devtmpfs + - fusectl + - hugetlbfs + - iso9660 + - mqueue + - nsfs + - overlay + - proc + - procfs + - pstore + - rpc_pipefs + - securityfs + - selinuxfs + - squashfs + - sysfs + - tracefs + match_type: strict + kubeletstats: + auth_type: serviceAccount + collection_interval: 20s + endpoint: ${env:K8S_NODE_NAME}:10250 + node: '${env:K8S_NODE_NAME}' + # Required to work for all CSPs without an issue + insecure_skip_verify: true + k8s_api_config: + auth_type: serviceAccount + metrics: + k8s.pod.cpu.node.utilization: + enabled: true + k8s.container.cpu_limit_utilization: + enabled: true + k8s.pod.cpu_limit_utilization: + enabled: true + k8s.container.cpu_request_utilization: + enabled: true + k8s.container.memory_limit_utilization: + enabled: true + k8s.pod.memory_limit_utilization: + enabled: true + k8s.container.memory_request_utilization: + enabled: true + k8s.node.uptime: + enabled: true + k8s.node.cpu.usage: + enabled: true + k8s.pod.cpu.usage: + enabled: true + extra_metadata_labels: + - container.id + service: + pipelines: + logs: + receivers: [filelog] + processors: [batch, k8sattributes, resourcedetection/system, resourcedetection/eks, resourcedetection/gcp, resource/demo, resource/k8s, resource/cloud, attributes/k8s_logs_dataset] + exporters: [debug, elasticsearch] + metrics: + receivers: [hostmetrics, kubeletstats] + processors: [batch, k8sattributes, elasticinframetrics, resourcedetection/system, resource/demo, resourcedetection/eks, resourcedetection/gcp, resource/k8s, resource/cloud, attributes/dataset, resource/process] + exporters: [debug, elasticsearch] + telemetry: + metrics: + address: ${env:MY_POD_IP}:8888 diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-deployment.yaml similarity index 97% rename from kubernetes/elastic-helm/configmap-elastic.yaml rename to kubernetes/elastic-helm/configmap-deployment.yaml index dd1c50e35b..7cc4b822fa 100644 --- a/kubernetes/elastic-helm/configmap-elastic.yaml +++ b/kubernetes/elastic-helm/configmap-deployment.yaml @@ -49,6 +49,7 @@ data: - otlp/elastic processors: - batch + - resource receivers: - otlp metrics: @@ -57,6 +58,7 @@ data: - debug processors: - batch + - resource receivers: - httpcheck/frontendproxy - otlp diff --git a/kubernetes/elastic-helm/daemonset.yaml b/kubernetes/elastic-helm/daemonset.yaml index 3bd510af13..d5fab18413 100644 --- a/kubernetes/elastic-helm/daemonset.yaml +++ b/kubernetes/elastic-helm/daemonset.yaml @@ -9,48 +9,41 @@ presets: kubernetesAttributes: enabled: true +image: + repository: docker.elastic.co/beats/elastic-agent + tag: 8.15.0-SNAPSHOT +command: + name: "/usr/share/elastic-agent/elastic-agent" + extraArgs: ["otel", "-c", "/etc/elastic-agent/otel.yaml"] +livenessProbe: + httpGet: + port: 8888 + path: metrics +readinessProbe: + httpGet: + port: 8888 + path: metrics + +securityContext: + runAsUser: 0 + runAsGroup: 0 + extraEnvs: - - name: ELASTIC_APM_ENDPOINT + - name: ELASTIC_ENDPOINT valueFrom: secretKeyRef: - name: elastic-secret - key: elastic_apm_endpoint - - name: ELASTIC_APM_SECRET_TOKEN + name: elastic-secret-ds + key: elastic_endpoint + - name: ELASTIC_API_KEY valueFrom: secretKeyRef: - name: elastic-secret - key: elastic_apm_secret_token + name: elastic-secret-ds + key: elastic_api_key + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName -config: - extensions: - k8s_observer: - auth_type: serviceAccount - node: ${env:K8S_NODE_NAME} - observe_pods: true - exporters: - otlp/elastic: - endpoint: ${ELASTIC_APM_ENDPOINT} - compression: none - headers: - Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN} - receivers: - receiver_creator: - watch_observers: [ k8s_observer ] - receivers: - redis: - rule: type == "port" && pod.name matches "redis" - config: - collection_interval: 2s - service: - extensions: [health_check, k8s_observer] - pipelines: - traces: - processors: [batch] - exporters: [otlp/elastic] - metrics: - receivers: [ receiver_creator] - processors: [batch] - exporters: [otlp/elastic] - logs: - processors: [batch] - exporters: [otlp/elastic] \ No newline at end of file +configMap: + create: false + existingName: elastic-otelcol-agent-ds diff --git a/kubernetes/elastic-helm/values.yaml b/kubernetes/elastic-helm/deployment.yaml similarity index 100% rename from kubernetes/elastic-helm/values.yaml rename to kubernetes/elastic-helm/deployment.yaml