Big refactoring of the repository

.gitignore

@@ -1 +1,2 @@
 data
+.env

Makefile

@@ -0,0 +1,17 @@
+.PHONY: clean
+
+export UID := $(shell id -u)
+export GID := $(shell id -g)
+
+clean:
+	docker compose down --remove-orphans
+	git clean -fxd data/
+	rm -f .env
+
+.env:
+	@echo UID=$(UID) > $@
+	@echo GID=$(GID) >> $@
+
+reset: clean .env
+
+prepare: .env

carbon-clickhouse.conf

@@ -1,6 +1,6 @@
 [common]
 # Prefix for store all internal carbon-clickhouse graphs. Supported macroses: {host}
-metric-prefix = "carbon.agents.{host}"
+metric-prefix = "carbon.{host}"
 # Endpoint for store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port"
 metric-endpoint = "local"
 # Interval of storing internal metrics. Like CARBON_METRIC_INTERVAL
@@ -31,29 +31,29 @@ type = "points"
 table = "graphite_data"
 threads = 1
 url = "http://clickhouse:8123/"
-timeout = "1m0s"
+timeout = "2m0s"
 
 [upload.graphite_reverse]
 type = "points-reverse"
 table = "graphite_reverse"
 threads = 1
 url = "http://clickhouse:8123/"
-timeout = "1m0s"
+timeout = "2m0s"
 
 [upload.graphite_index]
 type = "index"
 table = "graphite_index"
 threads = 1
 url = "http://clickhouse:8123/"
-timeout = "1m0s"
+timeout = "2m0s"
 cache-ttl = "12h0m0s"
 
 [upload.graphite_tagged]
 type = "tagged"
 table = "graphite_tagged"
 threads = 1
 url = "http://clickhouse:8123/"
-timeout = "1m0s"
+timeout = "2m0s"
 cache-ttl = "12h0m0s"
 
 [udp]

carbonapi.yml

@@ -0,0 +1,132 @@
+# Need to be URL, http or https
+# This url specifies the backend or a loadbalancer
+#
+# If you are using carbonzipper you should set it to
+# zipper's url
+#
+# If you are using plain go-carbon or graphite-clickhouse
+# you should set it to URL of go-carbon's carbonserver module
+# or graphite-clickhouse's http url.
+# Listen address, should always include hostname or ip address and a port.
+listen: ":7080"
+# Controls headers that would be passed to the backend
+headersToPass:
+  - "X-Dashboard-Id"
+  - "X-Grafana-Org-Id"
+  - "X-Panel-Id"
+# Max concurrent requests to CarbonZipper
+concurency: 1000
+cache:
+  # Type of caching. Valid: "mem", "memcache", "null"
+  type: "mem"
+  # Cache limit in megabytes
+  size_mb: 0
+  # Default cache timeout value. Identical to DEFAULT_CACHE_DURATION in graphite-web.
+  defaultTimeoutSec: 60
+# Amount of CPUs to use. 0 - unlimited
+cpus: 0
+# Timezone, default - local
+tz: ""
+
+# By default, functions like aggregate inherit tags from first series (for compatibility with graphite-web)
+# If set to true, tags are extracted from seriesByTag arguments
+#extractTagsFromArgs: false
+maxBatchSize: 0
+graphite:
+  # Host:port where to send internal metrics
+  # Empty = disabled
+  host: "carbon-clickhouse:2003"
+  interval: "60s"
+  prefix: "carbonapi"
+  # rules on how to construct metric name. For now only {prefix} and {fqdn} is supported.
+  # {prefix} will be replaced with the content of {prefix}
+  # {fqdn} will be repalced with fqdn
+  pattern: "{prefix}.{fqdn}"
+# Maximium idle connections to carbonzipper
+idleConnections: 1000
+pidFile: ""
+# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation
+upstreams:
+  # Number of 100ms buckets to track request distribution in. Used to build
+  # 'carbon.zipper.hostname.requests_in_0ms_to_100ms' metric and friends.
+  # Requests beyond the last bucket are logged as slow (default of 10 implies
+  # "slow" is >1 second).
+  # The last bucket is _not_ called 'requests_in_Xms_to_inf' on purpose, so
+  # we can change our minds about how many buckets we want to have and have
+  # their names remain consistent.
+  buckets: 10
+  timeouts:
+    find: "10s"
+    render: "50s"
+    connect: "200ms"
+  # Number of concurrent requests to any given backend - default is no limit.
+  # If set, you likely want >= MaxIdleConnsPerHost
+  concurrencyLimitPerServer: 1000
+
+  # Configures how often keep alive packets will be sent out
+  keepAliveInterval: "30s"
+
+  # Control http.MaxIdleConnsPerHost. Large values can lead to more idle
+  # connections on the backend servers which may bump into limits; tune with care.
+  maxIdleConnsPerHost: 1000
+  backendsv2:
+    backends:
+      - groupName: "clickhouse-cluster1"
+        # supported:
+        #    carbonapi_v2_pb - carbonapi 0.11 or earlier version of protocol.
+        #    carbonapi_v3_pb - new protocol, http interface (native)
+        #    carbonapi_v3_grpc - new protocol, gRPC interface (native)
+        #    protobuf, pb, pb3 - same as carbonapi_v2_pb
+        #    msgpack - protocol used by graphite-web 1.1 and metrictank
+        #    auto - carbonapi will do it's best to guess if it's carbonapi_v3_pb or carbonapi_v2_pb
+        #
+        #  non-native protocols will be internally converted to new protocol, which will increase memory consumption
+        protocol: "carbonapi_v3_pb"
+        # supported:
+        #    "broadcast" - send request to all backends in group and merge responses. This was default behavior for carbonapi 0.11 or earlier
+        #    "roundrobin" - send request to one backend.
+        #    "all - same as "broadcast"
+        #    "rr" - same as "roundrobin"
+        lbMethod: "rr"
+        # amount of retries in case of unsuccessful request
+        maxTries: 3
+        # amount of metrics per fetch request. Default: 0 - unlimited. If not specified, global will be used
+        maxBatchSize: 0
+        # interval for keep-alive http packets. If not specified, global will be used
+        keepAliveInterval: "30s"
+        # override for global concurrencyLimit.
+        concurrencyLimit: 1000
+        # override for global maxIdleConnsPerHost
+        maxIdleConnsPerHost: 1000
+        # per-group timeout override. If not specified, global will be used.
+        # Please note that ONLY min(global, local) will be used.
+        timeouts:
+          # Maximum backend request time for find requests.
+          find: "10s"
+          # Maximum backend request time for render requests. This is total one and doesn't take into account in-flight requests.
+          render: "50s"
+          # Timeout to connect to the server
+          connect: "200ms"
+        servers:
+          - "http://graphite-clickhouse:9090"
+# If not zero, enabled cache for find requests
+# This parameter controls when it will expire (in seconds)
+# Default: 600 (10 minutes)
+graphTemplates: /etc/graphTemplates.yaml
+expireDelaySec: 10
+# Uncomment this to get the behavior of graphite-web as proposed in https://github.com/graphite-project/graphite-web/pull/2239
+# Beware this will make darkbackground graphs less readable
+# defaultColors:
+#   "red": "ff0000"
+#   "green": "00ff00"
+#   "blue": "#0000ff"
+#   "darkred": "#c80032"
+#   "darkgreen": "00c800"
+#   "darkblue": "002173"
+logger:
+  - logger: ""
+    file: "stderr"
+    level: "debug"
+    encoding: "console"
+    encodingTime: "iso8601"
+    encodingDuration: "seconds"

docker-compose.yaml

@@ -1,44 +1,63 @@
-version: "3.5"
+---
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:21.8
-    user: "${UID:?You must do 'export UID' to launch}"
+    image: clickhouse/clickhouse-server:24.8
+    user: "${UID:?run `make prepare`}:${GID:?run `make prepare`}"
     volumes:
-    - "./rollup.xml:/etc/clickhouse-server/config.d/rollup.xml"
-    - "./init.sql:/docker-entrypoint-initdb.d/init.sql"
-    - "./data/clickhouse:/var/lib/clickhouse"
+      - "./rollup.xml:/etc/clickhouse-server/config.d/rollup.xml"
+      - "./init.sql:/docker-entrypoint-initdb.d/init.sql"
+      - "./data/clickhouse:/var/lib/clickhouse"
     networks:
       - graphite-clickhouse-tldr
+    cap_add: &caps
+      - SYS_PTRACE
+      - NET_ADMIN
+      - IPC_LOCK
+      - SYS_NICE
   carbon-clickhouse:
-    image: ghcr.io/lomik/carbon-clickhouse:0.11.2
-    user: "${UID:?You must do 'export UID' to launch}"
+    image: ghcr.io/go-graphite/carbon-clickhouse:0.11.8
+    user: "${UID:?run `make prepare`}:${GID:?run `make prepare`}"
     volumes:
-    - "./data:/data"
-    - "./carbon-clickhouse.conf:/etc/carbon-clickhouse/carbon-clickhouse.conf"
+      - "./data:/data"
+      - "./carbon-clickhouse.conf:/etc/carbon-clickhouse/carbon-clickhouse.conf"
     ports:
-    - "2003:2003" # plain tcp
-    - "2003:2003/udp" # plain udp
-    - "2004:2004" # pickle
-    - "2006:2006" # prometheus remote write
+      - "2003:2003"  # plain tcp
+      - "2003:2003/udp"  # plain udp
+      - "2004:2004"  # pickle
+      - "2006:2006"  # prometheus remote write
     networks:
       - graphite-clickhouse-tldr
   graphite-clickhouse:
-    image: ghcr.io/lomik/graphite-clickhouse:0.13.2
+    image: ghcr.io/go-graphite/graphite-clickhouse:0.14.0
     volumes:
-    - "./rollup.xml:/etc/graphite-clickhouse/rollup.xml"
-    - "./graphite-clickhouse.conf:/etc/graphite-clickhouse/graphite-clickhouse.conf"
+      - "./rollup.xml:/etc/graphite-clickhouse/rollup.xml"
+      - "./graphite-clickhouse.conf:/etc/graphite-clickhouse/graphite-clickhouse.conf"
+    ports:
+      - "9090:9090"  # carbonapi/graphite-web endpoint
+      - "9092:9092"  # prometheus endpoint
     networks:
       - graphite-clickhouse-tldr
-  graphite-web:
-    image: graphiteapp/graphite-statsd:1.1.8-1
+  carbonapi:
+    image: ghcr.io/go-graphite/carbonapi:0.17.0
+    volumes:
+      - "./carbonapi.yml:/etc/carbonapi.yml"
+      - "./graphTemplates.yaml:/etc/graphTemplates.yaml"
+    networks:
+      - graphite-clickhouse-tldr
+    ports:
+      - "7080:7080"  # http ui
+  grafana:
+    user: "${UID:?run `make prepare`}:${GID:?run `make prepare`}"
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: "addmin"
+    image: grafana/grafana:11.3.0-ubuntu
     volumes:
-    - "./graphiteweb_entrypoint.sh:/entrypoint.sh"
-    - "./graphiteweb.conf:/opt/graphite/webapp/graphite/local_settings.py"
-    entrypoint: /entrypoint.sh
+      - "./data/grafana:/var/lib/grafana"
+      - "./grafana-graphite-datasource.yaml:/etc/grafana/provisioning/datasources/graphite.yaml"
     networks:
       - graphite-clickhouse-tldr
     ports:
-    - "80:80" # http ui
+      - "80:3000"  # http ui
 
 networks:
   graphite-clickhouse-tldr:

grafana-graphite-datasource.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Graphite
+    type: graphite
+    access: proxy
+    url: http://carbonapi:7080
+    jsonData:
+      graphiteVersion: '1.1'
+

graphTemplates.yaml

@@ -0,0 +1,99 @@
+graphite:
+    bgColor: "white"
+    fgColor: "black"
+    colorList: ["blue", "green", "red", "purple", "brown", "yellow", "aqua", "grey", "magenta", "pink", "gold", "rose" ]
+
+graphite-dark:
+    colorList: ["blue", "green", "red", "purple", "brown", "yellow", "aqua", "grey", "magenta", "pink", "gold", "rose" ]
+
+solarized-light:
+    bgColor: "#fdf6e3"
+    fgColor: "#657b83"
+    majorLine: "#073642"
+    minorLine: "#586e75"
+    colorList: ["268bd2aa","859900aa","dc322faa","d33682aa","db4b16aa","b58900aa","2aa198aa","6c71c4aa"]
+
+solarized-dark:
+    bgColor: "#002b36"
+    fgColor: "#839496"
+    majorLine: "#fdf6e3"
+    minorLine: "#eee8d5"
+    colorList: ["268bd2aa","859900aa","dc322faa","d33682aa","db4b16aa","b58900aa","2aa198aa","6c71c4aa"]
+    fontName: "Sans"
+    fontSize: "10"
+    fontBold: false
+    fontItalic: false
+
+classic:
+    bgColor: "black"
+    fgColor: "white"
+    majorLine: "white"
+    minorLine: "grey"
+    colorList: ["blue","green","red","purple","brown","yellow","aqua","grey","magenta","pink","gold","rose"]
+    fontName: "Sans"
+    fontSize: "10"
+    fontBold: false
+    fontItalic: false
+
+noc:
+    bgColor: "black"
+    fgColor: "white"
+    majorLine: "white"
+    minorLine: "grey"
+    colorList: ["blue","green","red","yellow","purple","brown","aqua","grey","magenta","pink","gold","rose"]
+    fontName: "Sans"
+    fontSize: "10"
+    fontBold: false
+    fontItalic: false
+
+summary:
+    bgColor: "black"
+    colorList: ["#6666ff"," #66ff66"," #ff6666"]
+
+alphas:
+    bgColor: "white"
+    fgColor: "black"
+    majorLine: "grey"
+    minorLine: "rose"
+    colorList: ["00ff00aa","ff000077","00337799"]
+
+grafana:
+    bgColor: "white"
+    fgColor: "black"
+    minorLine: "grey"
+    majorLine: "rose"
+    colorList: ["#7eb26d","#eab839","#6ed0e0","#ef843c","#e24d42","#1f78c1","#ba43a9","#705da0","#508642","#cca300","#447ebc"]
+    fontName: "Sans"
+    fontSize: "10"
+    fontBold: false
+    fontItalic: false
+
+ocean1:
+    colorList: ["f7fcf0","e0f3db","ccebc5","a8ddb5","7bccc4","4eb3d3","2b8cbe","0868ac","084081"]
+
+ocean2:
+    colorList: ["084081","0868ac","2b8cbe","4eb3d3","7bccc4","a8ddb5","ccebc5","e0f3db","f7fcf0"]
+
+forest1:
+    colorList: ["f7fcf5","e5f5e0","c7e9c0","a1d99b","74c476","41ab5d","238b45","005a32"]
+
+forest2:
+    colorList: ["005a32","238b45","41ab5d","74c476","a1d99b","c7e9c0","e5f5e0","f7fcf5"]
+
+sunset1:
+    colorList: ["fff5eb","fee6ce","fdd0a2","fdae6b","fd8d3c","f16913","d94801","8c2d04"]
+
+sunset2:
+    colorList: ["8c2d04","d94801","f16913","fd8d3c","fdae6b","fdd0a2","fee6ce","fff5eb"]
+
+moonlight1:
+    colorList: ["fcfbfd","efedf5","dadaeb","bcbddc","9e9ac8","807dba","6a51a3","4a1486"]
+
+moonlight2:
+    colorList: ["4a1486","6a51a3","807dba","9e9ac8","bcbddc","dadaeb","efedf5","fcfbfd"]
+
+lava1:
+    colorList: ["fff5f0","fee0d2","fcbba1","fc9272","fb6a4a","ef3b2c","cb181d","99000d"]
+
+lava2:
+    colorList: ["99000d","cb181d","ef3b2c","fb6a4a","fc9272","fcbba1","fee0d2","fff5f0"]