From d8aea290deebd3bb8199b6ce72ab4a8ac3ce082d Mon Sep 17 00:00:00 2001 From: Mike Holloway Date: Fri, 28 Jun 2019 13:58:05 -0500 Subject: [PATCH] replace (brittle) promtail-based Docker logging with new loki-docker-driver --- README.md | 24 +++++++++--- ...tack.yaml => docker-compose-swarmstack.yml | 20 +--------- docker-compose.yaml | 37 ------------------- docker-compose.yml | 20 ++++++++++ promtail-docker-config.yaml | 22 ----------- up | 2 +- up-swarmstack | 2 +- 7 files changed, 42 insertions(+), 85 deletions(-) rename docker-compose-swarmstack.yaml => docker-compose-swarmstack.yml (56%) delete mode 100644 docker-compose.yaml create mode 100644 docker-compose.yml delete mode 100644 promtail-docker-config.yaml diff --git a/README.md b/README.md index 633bf43..e587da7 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,32 @@ Docker compose file for Grafana Loki, like Prometheus but for logs. Requires Doc ## INSTALLATION -On every one of your Docker nodes, create a directory and cron job that will populate container names: +On every Docker swarm node, first install Loki's Docker log driver plugin (it's important to install the plugin before modifying /etc/docker/daemon.json) ``` -mkdir -p /etc/promtail/targets -crontab -e +docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions ``` -and add the following line to your root cron (or cron of a user that can execute /bin/docker): +Next, add the following (at least the log-driver and log-opts below) to the file /etc/docker/daemon.json on each Docker swarm node: ``` -* * * * * docker ps --no-trunc --format '- targets: ["{{.ID}}"]\n labels:\n container_name: "{{.Names}}"' > /etc/promtail/targets/promtail-targets.yaml +{ + "debug" : false, + "log-driver": "loki", + "log-opts": { + "loki-url": "http://SWARM-FQDN-OR-IP:3100/api/prom/push", + "loki-batch-size": "400" + } +} ``` -Non-[swarmstack](https://github.com/swarmstack/swarmstack) users will need to edit promtail-docker-config.yaml and change the clients url from http://loki:3100/.. to the IP or DNS of one of your Docker nodes. +Finally, restart the Docker daemon on each swarm node (draining nodes first if necessary) to pick up the logging change: + +``` +docker node update --availbility drain node1.fqdn +systemctl restart docker +docker node update --availbility active node1.fqdn +``` ## USAGE diff --git a/docker-compose-swarmstack.yaml b/docker-compose-swarmstack.yml similarity index 56% rename from docker-compose-swarmstack.yaml rename to docker-compose-swarmstack.yml index 99e5b82..72cd302 100644 --- a/docker-compose-swarmstack.yaml +++ b/docker-compose-swarmstack.yml @@ -15,8 +15,6 @@ volumes: configs: loki_config: file: ./loki-local-config.yaml - promtail_config: - file: ./promtail-docker-config.yaml services: loki: @@ -35,21 +33,7 @@ services: constraints: - node.Labels.storagegroup==RED replicas: 1 + ports: + - "3100:3100" volumes: - loki-data:/data/loki - - promtail: - image: grafana/promtail:v0.1.0 - command: -config.file=/etc/promtail/promtail-docker-config.yaml - configs: - - source: promtail_config - target: /etc/promtail/promtail-docker-config.yaml - deploy: - labels: - prometheus.enable: "true" - prometheus.port: "9080" - prometheus.path: "/metrics" - mode: global - volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /etc/promtail/targets:/etc/promtail/targets:ro diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 2f37351..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,37 +0,0 @@ -version: "3.4" - -configs: - loki_config: - file: ./loki-local-config.yaml - promtail_config: - file: ./promtail-docker-config.yaml - -services: - loki: - image: grafana/loki:v0.1.0 - command: -config.file=/etc/loki/loki-local-config.yaml - configs: - - source: loki_config - target: /etc/loki/loki-local-config.yaml - deploy: - mode: replicated - replicas: 1 - ports: - - "3100:3100" - volumes: - - ./local_bind_volume_dir:/data/loki - - promtail: - image: grafana/promtail:v0.1.0 - command: -config.file=/etc/promtail/promtail-docker-config.yaml - configs: - - source: promtail_config - target: /etc/promtail/promtail-docker-config.yaml - deploy: - mode: global - # Open up port 9080 below only if you need to expose promtail's API externally - #ports: - # - "9080:9080" - volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /etc/promtail/targets:/etc/promtail/targets:ro diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4371193 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3.4" + +configs: + loki_config: + file: ./loki-local-config.yaml + +services: + loki: + image: grafana/loki:v0.1.0 + command: -config.file=/etc/loki/loki-local-config.yaml + configs: + - source: loki_config + target: /etc/loki/loki-local-config.yaml + deploy: + mode: replicated + replicas: 1 + ports: + - "3100:3100" + volumes: + - ./local_bind_volume_dir:/data/loki diff --git a/promtail-docker-config.yaml b/promtail-docker-config.yaml deleted file mode 100644 index 0c966c4..0000000 --- a/promtail-docker-config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -server: - http_listen_port: 9080 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/api/prom/push - -scrape_configs: -- job_name: docker - entry_parser: docker - file_sd_configs: - - files: - - /etc/promtail/targets/promtail-targets.yaml - relabel_configs: - - source_labels: [__address__] - target_label: container_id - - source_labels: [container_id] - target_label: __path__ - replacement: /var/lib/docker/containers/$1/*.log diff --git a/up b/up index 2d95344..34571ab 100755 --- a/up +++ b/up @@ -1,3 +1,3 @@ #!/bin/sh -docker stack deploy -c docker-compose.yaml loki +docker stack deploy -c docker-compose.yml loki diff --git a/up-swarmstack b/up-swarmstack index 9109806..5b44467 100755 --- a/up-swarmstack +++ b/up-swarmstack @@ -1,3 +1,3 @@ #!/bin/sh -docker stack deploy -c docker-compose-swarmstack.yaml loki +docker stack deploy -c docker-compose-swarmstack.yml loki