chore: add pyroscope dashboards (profile)

grafana/provisioning/dashboards/profile.json

@@ -0,0 +1,99 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 6,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "grafana-pyroscope-datasource",
+        "uid": "phlare"
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "targets": [
+        {
+          "datasource": {
+            "type": "grafana-pyroscope-datasource",
+            "uid": "phlare"
+          },
+          "groupBy": [],
+          "labelSelector": "{}",
+          "profileTypeId": "memory:alloc_space:bytes:space:bytes",
+          "queryType": "profile",
+          "refId": "A"
+        }
+      ],
+      "title": "Pyroscope alloc_space",
+      "type": "flamegraph"
+    },
+    {
+      "datasource": {
+        "type": "grafana-pyroscope-datasource",
+        "uid": "phlare"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "id": 1,
+      "targets": [
+        {
+          "datasource": {
+            "type": "grafana-pyroscope-datasource",
+            "uid": "phlare"
+          },
+          "groupBy": [],
+          "labelSelector": "{}",
+          "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds",
+          "queryType": "profile",
+          "refId": "A"
+        }
+      ],
+      "title": "Pyroscope CPU",
+      "type": "flamegraph"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Continuous Profiling",
+  "uid": "bd23ad18-c6c9-4f8a-be1d-4f5bacbf5cf9",
+  "version": 1,
+  "weekStart": ""
+}

grafana/provisioning/datasources/datasource.yml

@@ -58,3 +58,13 @@ datasources:
     isDefault: false
     uid: phlare
     version: 1
+
+  - name: Pyroscope
+    type: pyroscope
+    access: proxy
+    url: http://pyroscope:4040
+    basicAuth: false
+    editable: true
+    isDefault: false
+    uid: pyroscope
+    version: 1

localnet/start.sh

@@ -10,7 +10,7 @@ APP_CONFIG=$DATA_DIR/config/app.toml
 
 sed -i 's/prometheus = false/prometheus = true/g' $CONFIG
 # Change to 1s to have the same default configuration as v9
-sed -i 's/timeout_commit = "5s"/timeout_commit = "1s"/g' "$CONFIG"
+sed -i 's/timeout_commit = "5s"/timeout_commit = "3s"/g' "$CONFIG"
 # make sure the localhost IP is 0.0.0.0
 sed -i 's/pprof_laddr = "localhost:6060"/pprof_laddr = "0.0.0.0:6060"/g' "$CONFIG"
 sed -i 's/127.0.0.1/0.0.0.0/g' "$APP_CONFIG"

pyroscope/server.yml

@@ -0,0 +1,92 @@
+---
+# Disables analytics.
+analytics-opt-out: "false"
+
+# Log level: debug|info|warn|error.
+log-level: "info"
+
+# Log level: debug|info|warn|error.
+badger-log-level: "error"
+
+# Directory where pyroscope stores profiling data.
+storage-path: "/var/lib/pyroscope"
+
+# Port for the HTTP server used for data ingestion and web UI.
+api-bind-addr: ":4040"
+
+# Base URL for when the server is behind a reverse proxy with a different path.
+base-url: ""
+
+# Percentage of memory at which cache evictions start.
+cache-evict-threshold: "0.25"
+
+# Percentage of cache that is evicted per eviction run.
+cache-evict-volume: "0.33"
+
+# Database configuration. By default, Pyroscope Server uses embedded sqlite3 database.
+database:
+  # Indicates the database type. Supported DB engines: sqlite3.
+  type: "sqlite3"
+  # Database connection string. Specific to the engine.
+  url: "/var/lib/pyroscope/pyroscope.db"
+
+# Indicates whether value log files should be truncated to delete corrupt data, if any.
+badger-no-truncate: "false"
+
+# Disables /debug/pprof route.
+disable-pprof-endpoint: "false"
+
+# Max number of nodes used when saving profiles to disk.
+max-nodes-serialization: "2048"
+
+# Max number of nodes used to display data on the frontend.
+max-nodes-render: "8192"
+
+# Please don't use, this will soon be deprecated.
+hide-applications: []
+
+# Sets the maximum amount of time the profiling data is stored for. Data before this threshold is deleted. Disabled by default.
+retention: "0s"
+
+# Sets the maximum amount of time profile exemplars are stored for. Data before this threshold is deleted. Disabled by default.
+exemplars-retention: "0s"
+
+# Specifies how long the profiling data is stored per aggregation level. Disabled by default.
+retention-levels: {}
+# 0: 720h  # 30 days
+# 1: 2160h # 90 days
+
+# Metrics export rules.
+metrics-export-rules: {}
+
+# A list of scrape configurations.
+scrape-configs:
+  # The job name assigned to scraped profiles by default.
+  - job-name: pyroscope
+
+    # How frequently to scrape targets by default.
+    scrape-interval: 5s
+
+    # The list of profiles to be scraped from the targets.
+    enabled-profiles: [cpu, mem, goroutines, mutex, block]
+
+    # List of labeled statically configured targets for this job.
+    static-configs:
+      - application: single-node1
+        spy-name: gospy
+        targets:
+          - single-node1:6060
+        labels:
+          env: dev
+      - application: single-node2
+        spy-name: gospy
+        targets:
+          - single-node2:6060
+        labels:
+          env: dev          
+
+# Location of TLS Certificate file (.crt).
+tls-certificate-file: ""
+
+# Location of TLS Private key file (.key).
+tls-key-file: ""

single-node/docker-compose.yml

@@ -56,6 +56,7 @@ services:
     container_name: grafana
     depends_on:
       - prometheus
+      - pyroscope
     ports:
       - 8000:3000
     volumes:
@@ -124,3 +125,17 @@ services:
     restart: always
     deploy:
       mode: global
+
+  pyroscope:
+    image: "pyroscope/pyroscope:latest"
+    container_name: pyroscope
+    depends_on:
+      - single-node1    
+      - single-node2    
+    ports:
+      - "4040:4040"
+    volumes:
+      - ../pyroscope/:/etc/pyroscope/      
+    command:
+      - "server"
+    restart: always