From 197c8057fe6f6152d709f98a3d6bc2e0b1b7fa63 Mon Sep 17 00:00:00 2001 From: Matthew Hollick Date: Mon, 16 Sep 2019 22:45:53 +0100 Subject: [PATCH] improvements but still does not work right --- .gitignore | 1 + Dockerfiles/collectd/Dockerfile | 7 +++ carbon-clickhouse.conf | 74 ++++++++++++++++++++++++++ client.sh | 6 +++ collectd.conf | 94 +++++++++++++++++++++++++++++++++ docker-compose.yml | 57 +++++++++++--------- graphite-clickhouse.conf | 19 +++++++ init.sql | 28 ++++++++++ riemann.config | 23 +++----- riemann.config.logging | 50 ++++++++++++++++++ rollup.xml | 19 +++++++ 11 files changed, 336 insertions(+), 42 deletions(-) create mode 100644 .gitignore create mode 100644 Dockerfiles/collectd/Dockerfile create mode 100644 carbon-clickhouse.conf create mode 100755 client.sh create mode 100644 collectd.conf create mode 100644 graphite-clickhouse.conf create mode 100644 init.sql create mode 100644 riemann.config.logging create mode 100644 rollup.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3af0ccb --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/data diff --git a/Dockerfiles/collectd/Dockerfile b/Dockerfiles/collectd/Dockerfile new file mode 100644 index 0000000..88c4521 --- /dev/null +++ b/Dockerfiles/collectd/Dockerfile @@ -0,0 +1,7 @@ +FROM ubuntu + +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get install -y collectd + +CMD collectd -f diff --git a/carbon-clickhouse.conf b/carbon-clickhouse.conf new file mode 100644 index 0000000..5da761e --- /dev/null +++ b/carbon-clickhouse.conf @@ -0,0 +1,74 @@ +[common] +# Prefix for store all internal carbon-clickhouse graphs. Supported macroses: {host} +metric-prefix = "carbon.agents.{host}" +# Endpoint for store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port" +metric-endpoint = "local" +# Interval of storing internal metrics. Like CARBON_METRIC_INTERVAL +metric-interval = "1m0s" +# GOMAXPROCS +max-cpu = 1 + +[logging] +# "stderr", "stdout" can be used as file name +file = "stdout" +# Logging error level. Valid values: "debug", "info", "warn" "error" +level = "warn" + +[data] +# Folder for buffering received data +path = "/data/carbon-clickhouse/" +# Rotate (and upload) file interval. +# Minimize chunk-interval for minimize lag between point receive and store +chunk-interval = "1s" +# Auto-increase chunk interval if the number of unprocessed files is grown +# Sample, set chunk interval to 10 if unhandled files count >= 5 and set to 60s if unhandled files count >= 20: +# chunk-auto-interval = "5:10s,20:60s" +chunk-auto-interval = "5:5s,10:60s" + +[upload.graphite_reverse] +type = "points-reverse" +table = "graphite_reverse" +threads = 1 +url = "http://clickhouse:8123/" +timeout = "1m0s" + +[upload.graphite_index] +type = "index" +table = "graphite_index" +threads = 1 +url = "http://clickhouse:8123/" +timeout = "1m0s" +cache-ttl = "12h0m0s" + +[upload.graphite_tagged] +type = "tagged" +table = "graphite_tagged" +threads = 1 +url = "http://clickhouse:8123/" +timeout = "1m0s" +cache-ttl = "12h0m0s" + +[udp] +listen = ":2003" +enabled = true + +[tcp] +listen = ":2003" +enabled = true + +[pickle] +listen = ":2004" +enabled = true + +[prometheus] +listen = ":2006" +enabled = true + +# https://github.com/lomik/carbon-clickhouse/blob/master/grpc/carbon.proto +[grpc] +listen = ":2005" +enabled = false + +[pprof] +listen = "localhost:7007" +enabled = false diff --git a/client.sh b/client.sh new file mode 100755 index 0000000..1d57ad0 --- /dev/null +++ b/client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +docker-compose exec clickhouse bash -c " + export HOME=/var/lib/clickhouse/ + exec clickhouse client +" diff --git a/collectd.conf b/collectd.conf new file mode 100644 index 0000000..59509c8 --- /dev/null +++ b/collectd.conf @@ -0,0 +1,94 @@ +# +# Config file for collectd(1). +# Please read collectd.conf(5) for a list of options. +# http://collectd.org/ +# + +############################################################################## +# Global # +#----------------------------------------------------------------------------# +# Global settings for the daemon. # +############################################################################## + +FQDNLookup false + +#----------------------------------------------------------------------------# +# When enabled, plugins are loaded automatically with the default options # +# when an appropriate block is encountered. # +# Disabled by default. # +#----------------------------------------------------------------------------# +AutoLoadPlugin true + +#----------------------------------------------------------------------------# +# When enabled, internal statistics are collected, using "collectd" as the # +# plugin name. # +# Disabled by default. # +#----------------------------------------------------------------------------# +CollectInternalStats true + +#----------------------------------------------------------------------------# +# Interval at which to query values. This may be overwritten on a per-plugin # +# base by using the 'Interval' option of the LoadPlugin block: # +# # +# Interval 60 # +# # +#----------------------------------------------------------------------------# +Interval 60 + +# Limit the size of the write queue. Default is no limit. Setting up a limit is +# recommended for servers handling a high volume of traffic. +WriteQueueLimitHigh 1000000 +WriteQueueLimitLow 800000 + +############################################################################## +# Logging # +#----------------------------------------------------------------------------# +# Plugins which provide logging functions should be loaded first, so log # +# messages generated when loading or configuring other plugins can be # +# accessed. # +############################################################################## + +LoadPlugin logfile + + LogLevel err + File stdout + + +############################################################################## +# LoadPlugin section # +#----------------------------------------------------------------------------# +# Lines beginning with a single `#' belong to plugins which have been built # +# but are disabled by default. # +# # +# Lines beginning with `##' belong to plugins which have not been built due # +# to missing dependencies or because they have been deactivated explicitly. # +############################################################################## + +LoadPlugin cpu +LoadPlugin memory +LoadPlugin uptime + +############################################################################## +# Plugin configuration # +#----------------------------------------------------------------------------# +# In this section configuration stubs for each plugin are provided. A desc- # +# ription of those options is available in the collectd.conf(5) manual page. # +############################################################################## + + + ReportByCpu true + ReportByState true + ValuesPercentage true + + + + ValuesAbsolute true + ValuesPercentage true + + + + Property "metadata.broker.list" "kafka:9092" + + Format JSON + + diff --git a/docker-compose.yml b/docker-compose.yml index 965ae96..50bfd20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,17 +39,13 @@ services: - elasticsearch clickhouse: - image: yandex/clickhouse-server + image: yandex/clickhouse-server:19.6.2.11 hostname: clickhouse - ports: - - 8123:8123 - - 9000:9000 - - 9009:9009 - ulimits: - nproc: 65535 - nofile: - soft: 262144 - hard: 262144 + volumes: + - "./rollup.xml:/etc/clickhouse-server/config.d/rollup.xml" + - "./init.sql:/docker-entrypoint-initdb.d/init.sql" + - "./data/clickhouse/data:/var/lib/clickhouse/data" + - "./data/clickhouse/metadata:/var/lib/clickhouse/metadata" depends_on: - zookeeper @@ -78,27 +74,30 @@ services: depends_on: - elasticsearch - graphite-clickhouse: - image: mosquito/graphite-clickhouse - hostname: graphite-clickhouse - environment: - CLICKHOUSE_URL: http://clickhouse:8123 + carbon-clickhouse: + image: lomik/carbon-clickhouse:v0.10.2 + hostname: carbon-clickhouse + volumes: + - "./data/carbon-clickhouse:/data/carbon-clickhouse" + - "./carbon-clickhouse.conf:/etc/carbon-clickhouse/carbon-clickhouse.conf" ports: - - 9090:9090 + - "2003:2003" # plain tcp + - "2004:2004" # pickle + - "2006:2006" # prometheus remote write depends_on: - clickhouse - carbon-clickhouse: - image: mosquito/carbon-clickhouse - hostname: carbon-clickhouse - environment: - CLICKHOUSE_URL: http://clickhouse:8123 + graphite-clickhouse: + image: lomik/graphite-clickhouse:v0.11.1 + hostname: graphite-clickhouse + volumes: + - "./rollup.xml:/etc/graphite-clickhouse/rollup.xml" + - "./graphite-clickhouse.conf:/etc/graphite-clickhouse/graphite-clickhouse.conf" ports: - - 2003:2003 - - 2004:2004 - - 2005:2005 + - 9090:9090 depends_on: - clickhouse + - carbon-clickhouse carbonapi: image: mosquito/carbonapi @@ -138,4 +137,12 @@ services: image: docker.elastic.co/logstash/logstash:7.3.1 hostname: logstash-post depends_on: - - elasticsearch + - elasticsearch + + collectd: + image: collectd + hostname: collectd + volumes: + - ./collectd.conf:/etc/collectd/collectd.conf + depends_on: + - kafka diff --git a/graphite-clickhouse.conf b/graphite-clickhouse.conf new file mode 100644 index 0000000..02896fb --- /dev/null +++ b/graphite-clickhouse.conf @@ -0,0 +1,19 @@ +[common] +listen = ":9090" +max-cpu = 8 + +[clickhouse] +url = "http://clickhouse:8123/?max_query_size=2097152&readonly=2" +index-table = "graphite_index" +data-timeout = "1m0s" +index-timeout = "1m0s" +tagged-table = "graphite_tagged" + +[[data-table]] +table = "graphite_reverse" +reverse = true +rollup-conf = "/etc/graphite-clickhouse/rollup.xml" + +[logging] +file = "stdout" +level = "info" diff --git a/init.sql b/init.sql new file mode 100644 index 0000000..f62c7bf --- /dev/null +++ b/init.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS default.graphite_reverse ( + Path String, + Value Float64, + Time UInt32, + Date Date, + Timestamp UInt32 +) ENGINE = GraphiteMergeTree('graphite_rollup') +PARTITION BY toYYYYMM(Date) +ORDER BY (Path, Time); + +CREATE TABLE IF NOT EXISTS default.graphite_index ( + Date Date, + Level UInt32, + Path String, + Version UInt32 +) ENGINE = ReplacingMergeTree(Version) +PARTITION BY toYYYYMM(Date) +ORDER BY (Level, Path, Date); + +CREATE TABLE IF NOT EXISTS default.graphite_tagged ( + Date Date, + Tag1 String, + Path String, + Tags Array(String), + Version UInt32 +) ENGINE = ReplacingMergeTree(Version) +PARTITION BY toYYYYMM(Date) +ORDER BY (Tag1, Path, Date); diff --git a/riemann.config b/riemann.config index f3c3296..9944fe8 100644 --- a/riemann.config +++ b/riemann.config @@ -24,24 +24,13 @@ (expired (fn [event] (info "expired" event)))))) -(def elastic - (elasticsearch {:es-endpoint "http://elasticsearch:9200" - :es-index "riemann" - :index-suffix "-yyyy.MM.dd" - :type "event"} - (fn [event] - (merge event {})))) + +(def graph + (graphite {:host "carbon-clickhouse"})) (kafka-consumer {:consumer.config {:bootstrap.servers "kafka:9092" - :group.id "riemann"} - :topics ["program-logs"]}) + :group.id "collectd"} + :topics ["collectd"]}) (streams - elastic) - -;(defn- transmit -; [writer socket message] -; (let [bytes-out (bytes (byte-array (map byte message)))] -; (.write writer bytes-out 0 (count bytes-out)) -; (.flush writer))) - + graph) diff --git a/riemann.config.logging b/riemann.config.logging new file mode 100644 index 0000000..2901494 --- /dev/null +++ b/riemann.config.logging @@ -0,0 +1,50 @@ +; -*- mode: clojure; -*- +; vim: filetype=clojure + +(logging/init {:file "riemann.log"}) + +; Listen on the local interface over TCP (5555), UDP (5555), and websockets +; (5556) +(let [host "0.0.0.0"] + (tcp-server {:host host}) + (udp-server {:host host}) + (ws-server {:host host})) + +; Expire old events from the index every 5 seconds. +(periodically-expire 5) + +(let [index (index)] + ; Inbound events will be passed to these streams: + (streams + (default :ttl 60 + ; Index all events immediately. + index + + ; Log expired events. + (expired + (fn [event] (info "expired" event)))))) + +;(def elastic +; (elasticsearch {:es-endpoint "http://elasticsearch:9200" +; :es-index "riemann" +; :index-suffix "-yyyy.MM.dd" +; :type "event"} +; (fn [event] +; (merge event {})))) +; +;(kafka-consumer {:consumer.config {:bootstrap.servers "kafka:9092" +; :group.id "riemann"} +; :topics ["program-logs"]}) +; +;(streams +; elastic) + +(def graph + (graphite {:host "carbon-clickhouse"})) + +(kafka-consumer {:consumer.config {:bootstrap.servers "kafka:9092" + :group.id "collectd"} + :topics ["collectd"]}) + +(streams + graph) diff --git a/rollup.xml b/rollup.xml new file mode 100644 index 0000000..b3861a3 --- /dev/null +++ b/rollup.xml @@ -0,0 +1,19 @@ + + + Path + Time + Value + Timestamp + + avg + + 0 + 60 + + + 2592000 + 3600 + + + +