[opensearch] - Use opensearch exporter & remove dataprepper (#1356)

* use opensearch native exporter Signed-off-by: Pierre Tessier <[email protected]> * cleanup opensearch config Signed-off-by: Pierre Tessier <[email protected]> * fix opensearch datasource Signed-off-by: Pierre Tessier <[email protected]> * fix opensearch datasource Signed-off-by: Pierre Tessier <[email protected]> * use opensearch exporter Signed-off-by: Pierre Tessier <[email protected]> * remove dataprepper config files Signed-off-by: Pierre Tessier <[email protected]> * remove opensearch auth details Signed-off-by: Pierre Tessier <[email protected]> * opensearch memory limits Signed-off-by: Pierre Tessier <[email protected]> * opensearch memory limits Signed-off-by: Pierre Tessier <[email protected]> * remove dataprepper Signed-off-by: Pierre Tessier <[email protected]> --------- Signed-off-by: Pierre Tessier <[email protected]>
open-telemetry · Feb 2, 2024 · e15affc · e15affc
1 parent 26b1710
commit e15affc
Show file tree

Hide file tree

Showing 9 changed files with 41 additions and 60 deletions.
diff --git a/.env b/.env
@@ -6,10 +6,9 @@ IMAGE_NAME=ghcr.io/open-telemetry/demo
 
 # Dependent images
 COLLECTOR_CONTRIB_IMAGE=otel/opentelemetry-collector-contrib:0.93.0
-DATAPREPPER_IMAGE=opensearchproject/data-prepper:latest
 GRAFANA_IMAGE=grafana/grafana:10.3.1
 JAEGERTRACING_IMAGE=jaegertracing/all-in-one:1.53
-OPENSEARCH_IMAGE=opensearchproject/opensearch:latest
+OPENSEARCH_IMAGE=opensearchproject/opensearch:2.11.1
 POSTGRES_IMAGE=postgres:16.1
 PROMETHEUS_IMAGE=quay.io/prometheus/prometheus:v2.49.1
 REDIS_IMAGE=redis:7.2-alpine

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -42,6 +42,8 @@ the release.
   ([#1353](https://github.com/open-telemetry/opentelemetry-demo/pull/1353))
 * Update dependent services - Collector, Grafana, Jaeger, Prometheus, etc.
   ([#1354](https://github.com/open-telemetry/opentelemetry-demo/pull/1354))
+* [OpenSearch] Use native OpenSearch exporter from Collector
+  ([#1356](https://github.com/open-telemetry/opentelemetry-demo/pull/1356))
 
 ## 1.7.2
 

diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml
@@ -492,6 +492,7 @@ services:
       resources:
         limits:
           memory: 100M
+    restart: unless-stopped
     environment:
       - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
     volumes:
@@ -543,21 +544,28 @@ services:
       resources:
         limits:
           memory: 300M
+    restart: unless-stopped
     ports:
       - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
     logging: *logging
 
+  # OpenSearch
   opensearch:
     image: ${OPENSEARCH_IMAGE}
     container_name: opensearch
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    restart: unless-stopped
     environment:
       - cluster.name=demo-cluster
       - node.name=demo-node
       - bootstrap.memory_lock=true
       - discovery.type=single-node
-      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "DISABLE_INSTALL_DEMO_CONFIG=true"
-      - "DISABLE_SECURITY_PLUGIN=true"
+      - OPENSEARCH_JAVA_OPTS=-Xms300m -Xmx300m
+      - DISABLE_INSTALL_DEMO_CONFIG=true
+      - DISABLE_SECURITY_PLUGIN=true
     ulimits:
       memlock:
         soft: -1
@@ -566,14 +574,5 @@ services:
         soft: 65536
         hard: 65536
     ports:
-      - "9200:9200"
-    logging: *logging
-
-  dataprepper:
-    image: ${DATAPREPPER_IMAGE}
-    volumes:
-      - ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
-      - ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
-    ports:
-      - "21892"
+      - "9200"
     logging: *logging
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -645,6 +645,7 @@ services:
       resources:
         limits:
           memory: 100M
+    restart: unless-stopped
     environment:
       - "GF_INSTALL_PLUGINS=grafana-opensearch-datasource"
     volumes:
@@ -696,21 +697,28 @@ services:
       resources:
         limits:
           memory: 300M
+    restart: unless-stopped
     ports:
       - "${PROMETHEUS_SERVICE_PORT}:${PROMETHEUS_SERVICE_PORT}"
     logging: *logging
 
+  # OpenSearch
   opensearch:
     image: ${OPENSEARCH_IMAGE}
     container_name: opensearch
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    restart: unless-stopped
     environment:
       - cluster.name=demo-cluster
       - node.name=demo-node
       - bootstrap.memory_lock=true
       - discovery.type=single-node
-      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "DISABLE_INSTALL_DEMO_CONFIG=true"
-      - "DISABLE_SECURITY_PLUGIN=true"
+      - OPENSEARCH_JAVA_OPTS=-Xms300m -Xmx300m
+      - DISABLE_INSTALL_DEMO_CONFIG=true
+      - DISABLE_SECURITY_PLUGIN=true
     ulimits:
       memlock:
         soft: -1
@@ -719,16 +727,7 @@ services:
         soft: 65536
         hard: 65536
     ports:
-      - "9200:9200"
-    logging: *logging
-
-  dataprepper:
-    image: ${DATAPREPPER_IMAGE}
-    volumes:
-      - ./src/opensearch/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
-      - ./src/opensearch/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
-    ports:
-      - "21892"
+      - "9200"
     logging: *logging
 
   # *****

diff --git a/src/grafana/provisioning/dashboards/demo/demo-dashboard.json b/src/grafana/provisioning/dashboards/demo/demo-dashboard.json
@@ -456,7 +456,7 @@
               "type": "count"
             }
           ],
-          "query": "search source=otel\n| where serviceName=\"${service}\"\n| stats count() by severityText",
+          "query": "search source=otel\n| where resource.service.name=\"${service}\"\n| stats count() by severity.text",
           "queryType": "PPL",
           "refId": "A",
           "timeField": "time"
@@ -544,7 +544,7 @@
               "type": "count"
             }
           ],
-          "query": "search source=otel\n| where serviceName=\"${service}\"",
+          "query": "search source=otel\n| where resource.service.name=\"${service}\"",
           "queryType": "PPL",
           "refId": "A",
           "timeField": "time"

diff --git a/src/grafana/provisioning/datasources/opensearch.yaml b/src/grafana/provisioning/datasources/opensearch.yaml
@@ -11,10 +11,9 @@ datasources:
     editable: true
     isDefault: false
     jsonData:
-      database: 'otel'
-      interval: Daily
-      timeField: 'time'
-      version: '2.10.0'
-      flavor: 'opensearch'
+      database: otel
+      flavor: opensearch
+      logLevelField: severity
       logMessageField: body
-      logLevelField: severitynumber
+      pplEnabled: true
+      timeField: "@timestamp"
diff --git a/src/opensearch/data-prepper-config.yaml b/src/opensearch/data-prepper-config.yaml
diff --git a/src/opensearch/pipelines.yaml b/src/opensearch/pipelines.yaml
diff --git a/src/otelcollector/otelcol-config.yml b/src/otelcollector/otelcol-config.yml
@@ -1,7 +1,6 @@
 # Copyright The OpenTelemetry Authors
 # SPDX-License-Identifier: Apache-2.0
 
-
 receivers:
   otlp:
     protocols:
@@ -21,14 +20,16 @@ exporters:
     endpoint: "jaeger:4317"
     tls:
       insecure: true
-  otlp/logs:
-    endpoint: "dataprepper:21892"
-    tls:
-      insecure: true
   otlphttp/prometheus:
     endpoint: "http://prometheus:9090/api/v1/otlp"
     tls:
       insecure: true
+  opensearch:
+    logs_index: otel
+    http:
+      endpoint: "http://opensearch:9200"
+      tls:
+        insecure: true
 
 processors:
   batch:
@@ -64,4 +65,4 @@ service:
     logs:
       receivers: [otlp]
       processors: [batch]
-      exporters: [otlp/logs, debug]
+      exporters: [opensearch, debug]