[WIP] Grafana Loki

conf/grafana/provisioning/datasources.yml

@@ -0,0 +1,33 @@
+apiVersion: 1
+
+deleteDatasources:
+    - name: Graphite
+      orgId: 1  
+
+datasources:
+  - name: Loki
+    type: loki
+    url: http://loki:3100
+    jsonData:
+      maxLines: 1000
+      derivedFields:
+        # Field with internal link pointing to datasource in Grafana
+        - datasourceUid: my_jaeger_uid
+          matcherRegex: "traceID=(\\w+)"
+          name: TraceID
+          # url will be interpreted as query for the datasource
+          url: "$${__value.raw}"
+
+        # Field with external link
+        - matcherRegex: "traceID=(\\w+)"
+          name: TraceID
+          url: "http://jaeger:16686/trace/$${__value.raw}"
+
+  - name: InfluxDB - telegraf
+    type: influxdb
+    database: telegraf
+    user: telegraf
+    password: instrument
+    url: http://influxdb:8086
+    jsonData:
+      httpMode: POST

conf/telegraf/telegraf.conf

@@ -29,7 +29,7 @@
   username = "telegraf"
   password = "instrument"
   ## If true, the database tag will not be added to the metric.
-  exclude_database_tag = false
+  exclude_database_tag = true
   retention_policy = ""
   write_consistency = "any"
   timeout = "5s"
@@ -96,6 +96,18 @@
   endpoint = "unix:///var/run/docker.sock"
   timeout = "5s"
 
+  gather_services = false
+
+  container_name_include = []
+  container_name_exclude = []
+
+  docker_label_include = []
+  docker_label_exclude = []
+
+  perdevice = true
+
+  total = false
+
 # Statsd Server
 [[inputs.statsd]]
   ## Address and port to host UDP listener on
@@ -107,7 +119,7 @@
   ## Delete gauges every interval (default=false)
   delete_gauges = true
   ## Delete counters every interval (default=false)
-   delete_counters = true
+  delete_counters = true
 
   ## separator to use between elements of a statsd metric
   metric_separator = "_"

docker-compose.yml

@@ -1,28 +1,33 @@
-## Ultimate Instrumentation 
+## Ultimate Instrumentation
 ## Provides
 ##   - log collection and forwarding (logspout) (localhost:8000/logs)
 ##   - http/tcp/udp proxying (traefik) (localhost:8080/dashboard/)
 ##   - service discovery + dns + health-checks (consul) (localhost:8500/ui) (depends on registrator)
 ##   - time-series DB storage (influxdb) (localhost:8086)
-##   - metric & event visualization (grafana) (localhost:3000) 
-##   - distributed tracing (jaegar) (localhost:16686)
+##   - metric & event visualization (grafana) (localhost:3000)
+##   - distributed tracing (jaeger) (localhost:16686)
 ## http routing provided by traefik on `localhost/<route>` (see SERVICE_TAGS vars for routes)
 
 version: '3'
 
+# Create a network for intrastructure components
+networks:
+  infra:
+  monitor:
+  tracing:
+  instrument_web:
+    external: true
+
+# Create local persistent volumes
+# volumes:
+  # grafana-volume:
+  # influxdb-volume:
+
 services:
-  # logging
-  logspout:
-    image: "${REGISTRY}gliderlabs/logspout:master"
-    volumes: ['/var/run/docker.sock:/tmp/docker.sock:ro']
-    networks: ['infra','instrument_web']
-    ports: ['8000:80']  # logs avail at http://localhost:8000/logs
-    environment:
-      EXCLUDE_LABEL: logspout.exclude
-      # registrator
-      SERVICE_TAGS: 'traefik.enable=true,traefik.port=8000,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/logs`)'
 
-  # service discovery (consul + registrator)
+###############################################################################
+#                         SERVICE DISCOVERY                                   #
+###############################################################################
   consul:
     image: "${REGISTRY}consul:1.7"
     restart: always
@@ -34,27 +39,29 @@ services:
       # - '8301:8301/udp' # lan serf udp
       # - '8600:8600' # dns tcp
       # - '8600:8600/udp' # dns udp
-    environment: 
+    environment:
       # registrator
       SERVICE_TAGS: "traefik.enable=true,traefik.docker.network=instrument_web"
     labels:
-      - logspout.exclude 
+      - logspout.exclude
   registrator:
     image: "${REGISTRY}gliderlabs/registrator:master"
     depends_on: ['consul']
     networks: ['infra']
     volumes: ['/var/run/docker.sock:/tmp/docker.sock:ro']
     command: ["-internal=true", "consul://consul:8500"]
 
-  # load balancing 
+###############################################################################
+#                       LOAD BALANCE + PROXY                                  #
+###############################################################################
   traefik:
     image: "${REGISTRY}traefik:2.2"
     depends_on: ['registrator', 'consul']
     networks: ['infra','instrument_web']
-    ports: 
+    ports:
       - '8080:8080' # web ui
       - '80:80'
-    command: 
+    command:
       # - "--log.level=DEBUG"
       - "--api.insecure=true"
       - "--api.dashboard=true"
@@ -64,90 +71,105 @@ services:
       - "--providers.consulcatalog.cache=true"
       - "--providers.consulcatalog.exposedByDefault=false"
     labels:
-      - logspout.exclude 
+      - logspout.exclude
+
+###############################################################################
+#                            LOGGING                                          #
+###############################################################################
+  logspout:
+    image: "${REGISTRY}gliderlabs/logspout:master"
+    volumes: ['/var/run/docker.sock:/tmp/docker.sock:ro']
+    networks: ['infra','instrument_web']
+    ports: ['8000:80']  # logs avail at http://localhost:8000/logs
+    environment:
+      EXCLUDE_LABEL: logspout.exclude
+      # registrator
+      SERVICE_TAGS: 'traefik.enable=true,traefik.port=8000,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/logs`)'
+
+  promtail:
+    image:  "${REGISTRY}grafana/promtail:latest"
+    volumes:
+      - /var/log:/var/log:ro
+    command: ["-config.file=/etc/promtail/docker-config.yaml"]
+    networks: ['infra', 'monitor']
+    labels:
+      - logspout.exclude
 
-  #monitoring
-  ## TODO - get loki working (in grafana)
-  # loki:
-  #   image: ${REGISTRY}/grafana/loki:latest
-  #   depends_on: ['registrator', 'consul']
-  #   ports: ["3100:3100"]
-  #   command: ["-config.file=/etc/loki/local-config.yaml"]
-  #   networks: ['infra']
-  #   environment:
-  #     # LOGSPOUT: ignore
-  #     SERVICE_TAGS: "traefik.enable=true,traefik.http.routers.router0.rule=PathPrefix(`/loki`)"
+  loki:
+    image: "${REGISTRY}grafana/loki:latest"
+    depends_on: ['registrator', 'consul', 'telegraf']
+    ports: ["3100:3100"]
+    command: ["-config.file=/etc/loki/local-config.yaml"]
+    networks: ['infra', 'monitor']
+    labels:
+      - logspout.exclude
 
+###############################################################################
+#                         METRIC COLLECTION                                   #
+###############################################################################
   telegraf:
     image: "${REGISTRY}telegraf:1.14-alpine"
     restart: unless-stopped
     networks: ['infra', 'monitor']
     depends_on: ['influxdb']
     labels:
-      - logspout.exclude 
+      - logspout.exclude
     volumes:
       - ./conf/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro
       # For docker stats
       - /var/run/docker.sock:/var/run/docker.sock:ro
 
   influxdb:
     image: "${REGISTRY}influxdb:1.8-alpine"
-    restart: always
     ports: ['8086:8086']
     networks: ['monitor']
     labels:
-      - logspout.exclude 
-    environment: 
-      INFLUXDB_DB: telegraf 
+      - logspout.exclude
+    environment:
+      INFLUXDB_DB: telegraf
       INFLUXDB_USER: telegraf
       INFLUXDB_USER_PASSWORD: 'instrument'
       INFLUXDB_REPORTING_DISABLED: 'true'
     # volumes:
     #   - influxdb-volume:/var/lib/influxdb
 
+###############################################################################
+#                            VISUALIZATION                                    #
+###############################################################################
   grafana:
     image: "${REGISTRY}grafana/grafana:6.7.3"
-    depends_on: ['influxdb']
+    depends_on: ['influxdb', 'loki', 'jaeger']
     ports: ["3000:3000"]
     networks: ['monitor', 'instrument_web']
     user: "0"
     labels:
-      - logspout.exclude 
+      - logspout.exclude
+    volumes:
+      - ./conf/grafana/provisioning:/etc/grafana/provisioning:ro
     environment:
       GF_SECURITY_ADMIN_PASSWORD: 'instrument'
       GF_USERS_ALLOW_SIGN_UP: 'false'
       # registrator
       SERVICE_TAGS: 'traefik.enable=true,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/grafana`)'
 
-  # tracing
+###############################################################################
+#                         DISTRIBUTED TRACING                                 #
+###############################################################################
   jaeger:
     image: "${REGISTRY}jaegertracing/all-in-one:1.17"
     networks: ['monitor','tracing', 'instrument_web']
     environment:
       COLLECTOR_ZIPKIN_HTTP_PORT: 9411
     labels:
-      - logspout.exclude 
-    environment: 
+      - logspout.exclude
+    environment:
       # registrator
       SERVICE_TAGS: 'traefik.enable=true,traefik.docker.network=instrument_web,traefik.http.routers.router0.rule=PathPrefix(`/jaeger`)'
     ports:
       - '9411:9411'  # zipkin
-      - '16686:16686' # web ui  
+      - '16686:16686' # web ui
       # - '5778:5778' # http serve configs
       - '6831:6831/udp' # jaeger.thrift (compact)
       # - '6832:6832/udp' # jaeger.thrift (binary)
       # - '14268:14268' # http accept jaeger.thrift directly
-      # - '14250:14250' # http accept model.proto
-
-# Create a network for intrastructure components
-networks:
-  infra:
-  monitor:
-  tracing:
-  instrument_web:
-    external: true
-
-# Create local persistent volumes
-# volumes:
-  # grafana-volume:
-  # influxdb-volume:
+      # - '14250:14250' # http accept model.proto