Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Grafana Loki #11

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file.
33 changes: 33 additions & 0 deletions conf/grafana/provisioning/datasources.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
apiVersion: 1

deleteDatasources:
- name: Graphite
orgId: 1

datasources:
- name: Loki
type: loki
url: http://loki:3100
jsonData:
maxLines: 1000
derivedFields:
# Field with internal link pointing to datasource in Grafana
- datasourceUid: my_jaeger_uid
matcherRegex: "traceID=(\\w+)"
name: TraceID
# url will be interpreted as query for the datasource
url: "$${__value.raw}"

# Field with external link
- matcherRegex: "traceID=(\\w+)"
name: TraceID
url: "http://jaeger:16686/trace/$${__value.raw}"

- name: InfluxDB - telegraf
type: influxdb
database: telegraf
user: telegraf
password: instrument
url: http://influxdb:8086
jsonData:
httpMode: POST
16 changes: 14 additions & 2 deletions conf/telegraf/telegraf.conf
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
username = "telegraf"
password = "instrument"
## If true, the database tag will not be added to the metric.
exclude_database_tag = false
exclude_database_tag = true
retention_policy = ""
write_consistency = "any"
timeout = "5s"
Expand Down Expand Up @@ -96,6 +96,18 @@
endpoint = "unix:///var/run/docker.sock"
timeout = "5s"

gather_services = false

container_name_include = []
container_name_exclude = []

docker_label_include = []
docker_label_exclude = []

perdevice = true

total = false

# Statsd Server
[[inputs.statsd]]
## Address and port to host UDP listener on
Expand All @@ -107,7 +119,7 @@
## Delete gauges every interval (default=false)
delete_gauges = true
## Delete counters every interval (default=false)
delete_counters = true
delete_counters = true

## separator to use between elements of a statsd metric
metric_separator = "_"
Expand Down
134 changes: 78 additions & 56 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,28 +1,33 @@
## Ultimate Instrumentation
## Ultimate Instrumentation
## Provides
## - log collection and forwarding (logspout) (localhost:8000/logs)
## - http/tcp/udp proxying (traefik) (localhost:8080/dashboard/)
## - service discovery + dns + health-checks (consul) (localhost:8500/ui) (depends on registrator)
## - time-series DB storage (influxdb) (localhost:8086)
## - metric & event visualization (grafana) (localhost:3000)
## - distributed tracing (jaegar) (localhost:16686)
## - metric & event visualization (grafana) (localhost:3000)
## - distributed tracing (jaeger) (localhost:16686)
## http routing provided by traefik on `localhost/<route>` (see SERVICE_TAGS vars for routes)

version: '3'

# Create a network for intrastructure components
networks:
infra:
monitor:
tracing:
instrument_web:
external: true

# Create local persistent volumes
# volumes:
# grafana-volume:
# influxdb-volume:

services:
# logging
logspout:
image: "${REGISTRY}gliderlabs/logspout:master"
volumes: ['/var/run/docker.sock:/tmp/docker.sock:ro']
networks: ['infra','instrument_web']
ports: ['8000:80'] # logs avail at http://localhost:8000/logs
environment:
EXCLUDE_LABEL: logspout.exclude
# registrator
SERVICE_TAGS: 'traefik.enable=true,traefik.port=8000,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/logs`)'

# service discovery (consul + registrator)
###############################################################################
# SERVICE DISCOVERY #
###############################################################################
consul:
image: "${REGISTRY}consul:1.7"
restart: always
Expand All @@ -34,27 +39,29 @@ services:
# - '8301:8301/udp' # lan serf udp
# - '8600:8600' # dns tcp
# - '8600:8600/udp' # dns udp
environment:
environment:
# registrator
SERVICE_TAGS: "traefik.enable=true,traefik.docker.network=instrument_web"
labels:
- logspout.exclude
- logspout.exclude
registrator:
image: "${REGISTRY}gliderlabs/registrator:master"
depends_on: ['consul']
networks: ['infra']
volumes: ['/var/run/docker.sock:/tmp/docker.sock:ro']
command: ["-internal=true", "consul://consul:8500"]

# load balancing
###############################################################################
# LOAD BALANCE + PROXY #
###############################################################################
traefik:
image: "${REGISTRY}traefik:2.2"
depends_on: ['registrator', 'consul']
networks: ['infra','instrument_web']
ports:
ports:
- '8080:8080' # web ui
- '80:80'
command:
command:
# - "--log.level=DEBUG"
- "--api.insecure=true"
- "--api.dashboard=true"
Expand All @@ -64,90 +71,105 @@ services:
- "--providers.consulcatalog.cache=true"
- "--providers.consulcatalog.exposedByDefault=false"
labels:
- logspout.exclude
- logspout.exclude

###############################################################################
# LOGGING #
###############################################################################
logspout:
image: "${REGISTRY}gliderlabs/logspout:master"
volumes: ['/var/run/docker.sock:/tmp/docker.sock:ro']
networks: ['infra','instrument_web']
ports: ['8000:80'] # logs avail at http://localhost:8000/logs
environment:
EXCLUDE_LABEL: logspout.exclude
# registrator
SERVICE_TAGS: 'traefik.enable=true,traefik.port=8000,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/logs`)'

promtail:
image: "${REGISTRY}grafana/promtail:latest"
volumes:
- /var/log:/var/log:ro
command: ["-config.file=/etc/promtail/docker-config.yaml"]
networks: ['infra', 'monitor']
labels:
- logspout.exclude

#monitoring
## TODO - get loki working (in grafana)
# loki:
# image: ${REGISTRY}/grafana/loki:latest
# depends_on: ['registrator', 'consul']
# ports: ["3100:3100"]
# command: ["-config.file=/etc/loki/local-config.yaml"]
# networks: ['infra']
# environment:
# # LOGSPOUT: ignore
# SERVICE_TAGS: "traefik.enable=true,traefik.http.routers.router0.rule=PathPrefix(`/loki`)"
loki:
image: "${REGISTRY}grafana/loki:latest"
depends_on: ['registrator', 'consul', 'telegraf']
ports: ["3100:3100"]
command: ["-config.file=/etc/loki/local-config.yaml"]
networks: ['infra', 'monitor']
labels:
- logspout.exclude

###############################################################################
# METRIC COLLECTION #
###############################################################################
telegraf:
image: "${REGISTRY}telegraf:1.14-alpine"
restart: unless-stopped
networks: ['infra', 'monitor']
depends_on: ['influxdb']
labels:
- logspout.exclude
- logspout.exclude
volumes:
- ./conf/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro
# For docker stats
- /var/run/docker.sock:/var/run/docker.sock:ro

influxdb:
image: "${REGISTRY}influxdb:1.8-alpine"
restart: always
ports: ['8086:8086']
networks: ['monitor']
labels:
- logspout.exclude
environment:
INFLUXDB_DB: telegraf
- logspout.exclude
environment:
INFLUXDB_DB: telegraf
INFLUXDB_USER: telegraf
INFLUXDB_USER_PASSWORD: 'instrument'
INFLUXDB_REPORTING_DISABLED: 'true'
# volumes:
# - influxdb-volume:/var/lib/influxdb

###############################################################################
# VISUALIZATION #
###############################################################################
grafana:
image: "${REGISTRY}grafana/grafana:6.7.3"
depends_on: ['influxdb']
depends_on: ['influxdb', 'loki', 'jaeger']
ports: ["3000:3000"]
networks: ['monitor', 'instrument_web']
user: "0"
labels:
- logspout.exclude
- logspout.exclude
volumes:
- ./conf/grafana/provisioning:/etc/grafana/provisioning:ro
environment:
GF_SECURITY_ADMIN_PASSWORD: 'instrument'
GF_USERS_ALLOW_SIGN_UP: 'false'
# registrator
SERVICE_TAGS: 'traefik.enable=true,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/grafana`)'

# tracing
###############################################################################
# DISTRIBUTED TRACING #
###############################################################################
jaeger:
image: "${REGISTRY}jaegertracing/all-in-one:1.17"
networks: ['monitor','tracing', 'instrument_web']
environment:
COLLECTOR_ZIPKIN_HTTP_PORT: 9411
labels:
- logspout.exclude
environment:
- logspout.exclude
environment:
# registrator
SERVICE_TAGS: 'traefik.enable=true,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/jaeger`)'
ports:
- '9411:9411' # zipkin
- '16686:16686' # web ui
- '16686:16686' # web ui
# - '5778:5778' # http serve configs
- '6831:6831/udp' # jaeger.thrift (compact)
# - '6832:6832/udp' # jaeger.thrift (binary)
# - '14268:14268' # http accept jaeger.thrift directly
# - '14250:14250' # http accept model.proto

# Create a network for intrastructure components
networks:
infra:
monitor:
tracing:
instrument_web:
external: true

# Create local persistent volumes
# volumes:
# grafana-volume:
# influxdb-volume:
# - '14250:14250' # http accept model.proto