From a4b8462e251923ed09687ec2cefdeb70f409f33a Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Mon, 1 Jul 2024 09:54:50 +0200 Subject: [PATCH 1/9] add the Elastic Collector distribution --- .env.override | 7 ++++ .github/README.md | 6 ++- docker-compose.minimal.yml | 3 +- docker-compose.yml | 3 +- src/otelcollector/otelcol-config-extras.yml | 23 ----------- .../otelcol-elastic-config-extras.yaml | 41 +++++++++++++++++++ src/otelcollector/otelcol-elastic-config.yaml | 18 ++++++++ 7 files changed, 74 insertions(+), 27 deletions(-) create mode 100644 src/otelcollector/otelcol-elastic-config-extras.yaml create mode 100644 src/otelcollector/otelcol-elastic-config.yaml diff --git a/.env.override b/.env.override index 3c6c909469..53ffb483f9 100644 --- a/.env.override +++ b/.env.override @@ -10,3 +10,10 @@ IMAGE_NAME=ghcr.io/elastic/opentelemetry-demo # ********************* AD_SERVICE_DOCKERFILE=./src/adservice/Dockerfile.elastic FRAUD_SERVICE_DOCKERFILE=./src/frauddetectionservice/Dockerfile.elastic + +# ********************* +# Elastic Collector +# ********************* +COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-SNAPSHOT +OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml +OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml diff --git a/.github/README.md b/.github/README.md index 37d5f8e4b3..8c4a549863 100644 --- a/.github/README.md +++ b/.github/README.md @@ -7,15 +7,17 @@ The following guide describes how to setup the OpenTelemetry demo with Elastic O - The .NET agent within the [Cart service](../src/cartservice/src/Directory.Build.props) has been replaced with the Elastic distribution of the OpenTelemetry .NET Agent. You can find more information about the Elastic distribution in [this blog post](https://www.elastic.co/observability-labs/blog/elastic-opentelemetry-distribution-dotnet-applications). - The Elastic distribution of the OpenTelemetry Node.js Agent has replaced the OpenTelemetry Node.js agent in the [Payment service](../src/paymentservice/package.json). Additional details about the Elastic distribution are available in [this blog post](https://www.elastic.co/observability-labs/blog/elastic-opentelemetry-distribution-node-js). +Additionally, the OpenTelemetry Contrib collector has also been changed to the [Elastic OpenTelemetry Collector distribution](https://github.com/elastic/elastic-agent/blob/main/internal/pkg/otel/README.md). This ensures a more integrated and optimized experience with Elastic Observability. + ## Docker compose 1. Start a free trial on [Elastic Cloud](https://cloud.elastic.co/) and copy the `endpoint` and `secretToken` from the Elastic APM setup instructions in your Kibana. -1. Open the file `src/otelcollector/otelcol-config-extras.yml` in an editor and replace the following two placeholders: +1. Open the file `src/otelcollector/otelcol-elastic-config-extras.yaml` in an editor and replace the following two placeholders: - `YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX`: your Elastic APM endpoint (*without* `https://` prefix) that *must* also include the port (example: `1234567.apm.us-west2.gcp.elastic-cloud.com:443`). - `YOUR_APM_SECRET_TOKEN`: your Elastic APM secret token. 1. Start the demo with the following command from the repository's root directory: ``` - docker-compose up -d + make start ``` ## Kubernetes diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index 5720b4e0c8..648a4b55d4 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -582,7 +582,8 @@ services: limits: memory: 200M restart: unless-stopped - command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ] + command: ["otel", "--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ] + entrypoint: [ /usr/share/elastic-agent/elastic-agent ] volumes: - ${OTEL_COLLECTOR_CONFIG}:/etc/otelcol-config.yml - ${OTEL_COLLECTOR_CONFIG_EXTRAS}:/etc/otelcol-config-extras.yml diff --git a/docker-compose.yml b/docker-compose.yml index 511883c21f..f086d30b56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -692,7 +692,8 @@ services: limits: memory: 200M restart: unless-stopped - command: [ "--config=/etc/otelcol-config.yml", "--config=/etc/otelcol-config-extras.yml" ] + command: ["otel", "--config", "/etc/otelcol-config.yml", "--config", "/etc/otelcol-config-extras.yml" ] + entrypoint: [ /usr/share/elastic-agent/elastic-agent ] volumes: - ${OTEL_COLLECTOR_CONFIG}:/etc/otelcol-config.yml - ${OTEL_COLLECTOR_CONFIG_EXTRAS}:/etc/otelcol-config-extras.yml diff --git a/src/otelcollector/otelcol-config-extras.yml b/src/otelcollector/otelcol-config-extras.yml index ce2395d485..73dd011b2c 100644 --- a/src/otelcollector/otelcol-config-extras.yml +++ b/src/otelcollector/otelcol-config-extras.yml @@ -16,26 +16,3 @@ # pipelines: # traces: # exporters: [spanmetrics, otlphttp/example] - -exporters: - otlp/elastic: - # !!! Elastic APM https endpoint WITHOUT the "https://" prefix - endpoint: "YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX" - compression: none - headers: - Authorization: "Bearer YOUR_APM_SECRET_TOKEN" - -service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [spanmetrics, otlp/elastic] - metrics: - receivers: [otlp, spanmetrics] - processors: [batch] - exporters: [otlp/elastic] - logs: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] diff --git a/src/otelcollector/otelcol-elastic-config-extras.yaml b/src/otelcollector/otelcol-elastic-config-extras.yaml new file mode 100644 index 0000000000..ce2395d485 --- /dev/null +++ b/src/otelcollector/otelcol-elastic-config-extras.yaml @@ -0,0 +1,41 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# extra settings to be merged into OpenTelemetry Collector configuration +# do not delete this file + +## Example configuration for sending data to your own OTLP HTTP backend +## Note: the spanmetrics exporter must be included in the exporters array +## if overriding the traces pipeline. +## +# exporters: +# otlphttp/example: +# endpoint: +# +# service: +# pipelines: +# traces: +# exporters: [spanmetrics, otlphttp/example] + +exporters: + otlp/elastic: + # !!! Elastic APM https endpoint WITHOUT the "https://" prefix + endpoint: "YOUR_APM_ENDPOINT_WITHOUT_HTTPS_PREFIX" + compression: none + headers: + Authorization: "Bearer YOUR_APM_SECRET_TOKEN" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [spanmetrics, otlp/elastic] + metrics: + receivers: [otlp, spanmetrics] + processors: [batch] + exporters: [otlp/elastic] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlp/elastic] diff --git a/src/otelcollector/otelcol-elastic-config.yaml b/src/otelcollector/otelcol-elastic-config.yaml new file mode 100644 index 0000000000..63ec2b97bd --- /dev/null +++ b/src/otelcollector/otelcol-elastic-config.yaml @@ -0,0 +1,18 @@ +receivers: + otlp: + protocols: + grpc: + http: + cors: + allowed_origins: + - "http://*" + - "https://*" + +exporters: + debug: + +processors: + batch: + +connectors: + spanmetrics: From 3999c2f97a185b54fb4c3dcc4875965ab71c36c6 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Mon, 1 Jul 2024 16:43:27 +0200 Subject: [PATCH 2/9] feat: add Elastic distro to Helm deployment --- .../elastic-helm/configmap-elastic.yaml | 108 ++++++++++++++++++ kubernetes/elastic-helm/values.yaml | 55 ++++----- 2 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 kubernetes/elastic-helm/configmap-elastic.yaml diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-elastic.yaml new file mode 100644 index 0000000000..54c0485cfa --- /dev/null +++ b/kubernetes/elastic-helm/configmap-elastic.yaml @@ -0,0 +1,108 @@ +--- +# Source: opentelemetry-demo/charts/opentelemetry-collector/templates/configmap-agent.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-otelcol-agent + namespace: default + labels: + app.kubernetes.io/name: otelcol + +data: + relay: | + connectors: + spanmetrics: {} + exporters: + debug: {} + otlp/elastic: + endpoint: ${ELASTIC_APM_ENDPOINT} + compression: none + headers: + Authorization: ApiKey ${ELASTIC_APM_SECRET_TOKEN} + extensions: + processors: + batch: {} + k8sattributes: + extract: + metadata: + - k8s.namespace.name + - k8s.deployment.name + - k8s.statefulset.name + - k8s.daemonset.name + - k8s.cronjob.name + - k8s.job.name + - k8s.node.name + - k8s.pod.name + - k8s.pod.uid + - k8s.pod.start_time + filter: + node_from_env_var: K8S_NODE_NAME + passthrough: false + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: connection + resource: + attributes: + - action: insert + from_attribute: k8s.pod.uid + key: service.instance.id + receivers: + httpcheck/frontendproxy: + targets: + - endpoint: http://example-frontendproxy:8080 + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 + http: + cors: + allowed_origins: + - http://* + - https://* + endpoint: ${env:MY_POD_IP}:4318 + service: + extensions: + pipelines: + logs: + exporters: + - debug + - otlp/elastic + processors: + - k8sattributes + - resource + - batch + receivers: + - otlp + metrics: + exporters: + - otlp/elastic + - debug + processors: + - k8sattributes + - resource + - batch + receivers: + - httpcheck/frontendproxy + - otlp + - spanmetrics + traces: + exporters: + - otlp/elastic + - debug + - spanmetrics + processors: + - k8sattributes + - resource + - batch + receivers: + - otlp + telemetry: + metrics: + address: ${env:MY_POD_IP}:8888 + diff --git a/kubernetes/elastic-helm/values.yaml b/kubernetes/elastic-helm/values.yaml index 616bbda572..d490086495 100644 --- a/kubernetes/elastic-helm/values.yaml +++ b/kubernetes/elastic-helm/values.yaml @@ -4,6 +4,12 @@ default: tag: 1.10.1 opentelemetry-collector: + image: + repository: docker.elastic.co/beats/elastic-agent + tag: 8.15.0-SNAPSHOT + command: + name: "/usr/share/elastic-agent/elastic-agent" + extraArgs: ["otel", "-c", "/etc/elastic-agent/otel.yaml"] mode: "deployment" presets: kubernetesAttributes: @@ -24,38 +30,23 @@ opentelemetry-collector: secretKeyRef: name: elastic-secret key: elastic_apm_secret_token - config: - exporters: - otlp/elastic: - endpoint: ${ELASTIC_APM_ENDPOINT} - compression: none - headers: - Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN} - receivers: - otlp: - protocols: - grpc: - endpoint: ${MY_POD_IP}:4317 - http: - cors: - allowed_origins: - - http://* - - https://* - endpoint: ${MY_POD_IP}:4318 - service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] - logs: - receivers: [otlp] - processors: [batch] - exporters: [otlp/elastic] + + configMap: + create: false + existingName: elastic-otelcol-agent + + livenessProbe: + httpGet: + port: 8888 + path: metrics + readinessProbe: + httpGet: + port: 8888 + path: metrics + +opensearch: + enabled: false + grafana: enabled: false From 2b523226c3f22e25439cb70e63f91c11a83262a6 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Tue, 2 Jul 2024 17:56:25 +0200 Subject: [PATCH 3/9] remove attributes processors for k8s config --- .../elastic-helm/configmap-elastic.yaml | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-elastic.yaml index 54c0485cfa..edca8c7986 100644 --- a/kubernetes/elastic-helm/configmap-elastic.yaml +++ b/kubernetes/elastic-helm/configmap-elastic.yaml @@ -1,5 +1,4 @@ --- -# Source: opentelemetry-demo/charts/opentelemetry-collector/templates/configmap-agent.yaml apiVersion: v1 kind: ConfigMap metadata: @@ -22,36 +21,6 @@ data: extensions: processors: batch: {} - k8sattributes: - extract: - metadata: - - k8s.namespace.name - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.cronjob.name - - k8s.job.name - - k8s.node.name - - k8s.pod.name - - k8s.pod.uid - - k8s.pod.start_time - filter: - node_from_env_var: K8S_NODE_NAME - passthrough: false - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - resource: - attributes: - - action: insert - from_attribute: k8s.pod.uid - key: service.instance.id receivers: httpcheck/frontendproxy: targets: @@ -74,8 +43,6 @@ data: - debug - otlp/elastic processors: - - k8sattributes - - resource - batch receivers: - otlp @@ -84,8 +51,6 @@ data: - otlp/elastic - debug processors: - - k8sattributes - - resource - batch receivers: - httpcheck/frontendproxy @@ -97,8 +62,6 @@ data: - debug - spanmetrics processors: - - k8sattributes - - resource - batch receivers: - otlp From b237bf4d41fec3be1ead54a42d84df7b92c04db2 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Tue, 2 Jul 2024 18:01:45 +0200 Subject: [PATCH 4/9] chore: bump elastic-agent snapshot version --- .env.override | 2 +- kubernetes/elastic-helm/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.override b/.env.override index 53ffb483f9..ad0661922a 100644 --- a/.env.override +++ b/.env.override @@ -14,6 +14,6 @@ FRAUD_SERVICE_DOCKERFILE=./src/frauddetectionservice/Dockerfile.elastic # ********************* # Elastic Collector # ********************* -COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-SNAPSHOT +COLLECTOR_CONTRIB_IMAGE=docker.elastic.co/beats/elastic-agent:8.15.0-7b611e39-SNAPSHOT OTEL_COLLECTOR_CONFIG=./src/otelcollector/otelcol-elastic-config.yaml OTEL_COLLECTOR_CONFIG_EXTRAS=./src/otelcollector/otelcol-elastic-config-extras.yaml diff --git a/kubernetes/elastic-helm/values.yaml b/kubernetes/elastic-helm/values.yaml index d490086495..c69df4980c 100644 --- a/kubernetes/elastic-helm/values.yaml +++ b/kubernetes/elastic-helm/values.yaml @@ -6,7 +6,7 @@ default: opentelemetry-collector: image: repository: docker.elastic.co/beats/elastic-agent - tag: 8.15.0-SNAPSHOT + tag: 8.15.0-7b611e39-SNAPSHOT command: name: "/usr/share/elastic-agent/elastic-agent" extraArgs: ["otel", "-c", "/etc/elastic-agent/otel.yaml"] From 293606e66ac6bd48a907636b362f84283814c058 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Tue, 2 Jul 2024 18:05:40 +0200 Subject: [PATCH 5/9] docs: add k8s configuration deployment command --- .github/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/README.md b/.github/README.md index 8c4a549863..4958b2bd73 100644 --- a/.github/README.md +++ b/.github/README.md @@ -48,6 +48,9 @@ Additionally, the OpenTelemetry Contrib collector has also been changed to the [ # !(when an older helm open-telemetry repo exists) update the open-telemetry helm repo helm repo update open-telemetry + # deploy the configuration for the Elastic OpenTelemetry collector distribution + kubectl apply -f configmap-elastic.yaml + # deploy the demo through helm install helm install -f values.yaml my-otel-demo open-telemetry/opentelemetry-demo ``` From a960d81f68d1614ec1df3638bfdb5eeeaa3f5413 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Tue, 2 Jul 2024 18:13:04 +0200 Subject: [PATCH 6/9] chore: remove otel header from Elastic config --- .../otelcol-elastic-config-extras.yaml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/otelcollector/otelcol-elastic-config-extras.yaml b/src/otelcollector/otelcol-elastic-config-extras.yaml index ce2395d485..530d47ad15 100644 --- a/src/otelcollector/otelcol-elastic-config-extras.yaml +++ b/src/otelcollector/otelcol-elastic-config-extras.yaml @@ -1,22 +1,3 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# extra settings to be merged into OpenTelemetry Collector configuration -# do not delete this file - -## Example configuration for sending data to your own OTLP HTTP backend -## Note: the spanmetrics exporter must be included in the exporters array -## if overriding the traces pipeline. -## -# exporters: -# otlphttp/example: -# endpoint: -# -# service: -# pipelines: -# traces: -# exporters: [spanmetrics, otlphttp/example] - exporters: otlp/elastic: # !!! Elastic APM https endpoint WITHOUT the "https://" prefix From 1b54ec20630edd80854179adac771bf0eb46ba6c Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Tue, 2 Jul 2024 18:21:39 +0200 Subject: [PATCH 7/9] use Bearer for apm authorization header --- kubernetes/elastic-helm/configmap-elastic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-elastic.yaml index edca8c7986..b6cde781e3 100644 --- a/kubernetes/elastic-helm/configmap-elastic.yaml +++ b/kubernetes/elastic-helm/configmap-elastic.yaml @@ -17,7 +17,7 @@ data: endpoint: ${ELASTIC_APM_ENDPOINT} compression: none headers: - Authorization: ApiKey ${ELASTIC_APM_SECRET_TOKEN} + Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN} extensions: processors: batch: {} From 44618786074dcdfa580e50726ba9563bf8fce1f5 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Wed, 3 Jul 2024 12:46:45 +0200 Subject: [PATCH 8/9] Update kubernetes/elastic-helm/values.yaml Co-authored-by: Chris Mark --- kubernetes/elastic-helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/elastic-helm/values.yaml b/kubernetes/elastic-helm/values.yaml index c69df4980c..d490086495 100644 --- a/kubernetes/elastic-helm/values.yaml +++ b/kubernetes/elastic-helm/values.yaml @@ -6,7 +6,7 @@ default: opentelemetry-collector: image: repository: docker.elastic.co/beats/elastic-agent - tag: 8.15.0-7b611e39-SNAPSHOT + tag: 8.15.0-SNAPSHOT command: name: "/usr/share/elastic-agent/elastic-agent" extraArgs: ["otel", "-c", "/etc/elastic-agent/otel.yaml"] From 99881314655924be832ca14e6a363974d9356182 Mon Sep 17 00:00:00 2001 From: Roger Coll Date: Wed, 3 Jul 2024 12:47:13 +0200 Subject: [PATCH 9/9] Update kubernetes/elastic-helm/configmap-elastic.yaml Co-authored-by: Chris Mark --- kubernetes/elastic-helm/configmap-elastic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/elastic-helm/configmap-elastic.yaml b/kubernetes/elastic-helm/configmap-elastic.yaml index b6cde781e3..5799301baa 100644 --- a/kubernetes/elastic-helm/configmap-elastic.yaml +++ b/kubernetes/elastic-helm/configmap-elastic.yaml @@ -14,7 +14,7 @@ data: exporters: debug: {} otlp/elastic: - endpoint: ${ELASTIC_APM_ENDPOINT} + endpoint: ${env:ELASTIC_APM_ENDPOINT} compression: none headers: Authorization: Bearer ${ELASTIC_APM_SECRET_TOKEN}