Setup basic EFK stack

Dockerfile

@@ -29,13 +29,15 @@ ADD script /opt/pwn.college/script
 ADD ssh /opt/pwn.college/ssh
 ADD logging /opt/pwn.college/logging
 ADD nginx-proxy /opt/pwn.college/nginx-proxy
+ADD fluentd /opt/pwn.college/fluentd
 ADD challenge /opt/pwn.college/challenge
 ADD dojo_plugin /opt/CTFd/CTFd/plugins/dojo_plugin
 ADD dojo_theme /opt/CTFd/CTFd/themes/dojo_theme
 ADD data_example /opt/pwn.college/data_example
 ADD docker-compose.yml /opt/pwn.college/docker-compose.yml
 ADD docker-entrypoint.sh /opt/pwn.college/docker-entrypoint.sh
 
+ADD etc/docker /etc/docker
 ADD etc/ssh/sshd_config /etc/ssh/sshd_config
 ADD etc/systemd/system/pwn.college.service /etc/systemd/system/pwn.college.service
 ADD etc/systemd/system/pwn.college.logging.service /etc/systemd/system/pwn.college.logging.service

docker-compose.yml

@@ -44,6 +44,7 @@ services:
       - /opt/CTFd:/opt/CTFd:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
     depends_on:
+      - fluentd
       - db
 
   db:
@@ -59,13 +60,17 @@ services:
       - ./data/mysql:/var/lib/mysql
     # This command is required to set important mariadb defaults
     command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --wait_timeout=28800, --log-warnings=0]
+    depends_on:
+      - fluentd
 
   cache:
     container_name: ctfd_cache
     image: redis:4
     restart: always
     volumes:
       - ./data/redis:/data
+    depends_on:
+      - fluentd
 
   mailserver:
     container_name: mailserver
@@ -80,6 +85,8 @@ services:
       - ./data/dms/mail-state:/var/mail-state
       - ./data/dms/mail-logs:/var/log/mail
       - ./data/dms/config:/tmp/docker-mailserver
+    depends_on:
+      - fluentd
 
   nginx-proxy:
     container_name: nginx_proxy
@@ -100,6 +107,8 @@ services:
       - ./nginx-proxy/etc/passwd:/etc/passwd
       - ./data/homes:/var/homes:shared
       - /var/run/docker.sock:/tmp/${DOCKER_PSLR}/docker.sock:ro
+    depends_on:
+      - fluentd
 
   nginx-proxy-acme:
     container_name: nginx_proxy_acme
@@ -111,6 +120,42 @@ services:
       - certs:/etc/nginx/certs:rw
       - acme:/etc/acme.sh
       - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - fluentd
+
+  fluentd:
+    container_name: fluentd
+    build: fluentd
+    restart: always
+    volumes:
+      - ./fluentd/etc/fluent.conf:/fluentd/etc/fluent.conf
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200k"
+        max-file: "10"
+
+  elasticsearch:
+    container_name: elasticsearch
+    image: elasticsearch:8.1.2
+    restart: always
+    environment:
+      - xpack.security.enabled=false
+      - discovery.type=single-node
+    depends_on:
+      - fluentd
+
+  kibana:
+    container_name: kibana
+    image: kibana:8.1.2
+    restart: always
+    ports:
+      - "5601:5601"
+    depends_on:
+      - fluentd
 
 volumes:
   conf:

etc/docker/daemon.json

@@ -0,0 +1,7 @@
+{
+    "log-driver": "fluentd",
+    "log-opts": {
+        "fluentd-address": "localhost:24224",
+        "tag": "docker.{{.Name}}.{{.ID}}"
+    }
+}

fluentd/Dockerfile

@@ -0,0 +1,12 @@
+FROM fluent/fluentd:v1.15.2-debian-1.0
+USER root
+RUN gem install fluent-plugin-elasticsearch --no-document --version 5.2.3 \
+    && gem uninstall --ignore-dependencies faraday \
+    && gem install faraday --no-document --version 1.10.0 \
+    && gem sources --clear-all \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+USER fluent
+
+# TODO: https://github.com/uken/fluent-plugin-elasticsearch/issues/984#issuecomment-1239739636

fluentd/etc/fluent.conf

@@ -0,0 +1,26 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<match *.**>
+  @type copy
+
+  <store>
+    @type elasticsearch
+    host elasticsearch
+    port 9200
+    logstash_format true
+    logstash_prefix fluentd
+    logstash_dateformat %Y%m%d
+    include_tag_key true
+    type_name access_log
+    tag_key @log_name
+    flush_interval 1s
+  </store>
+
+  <store>
+    @type stdout
+  </store>
+</match>

run.sh

@@ -15,6 +15,7 @@ docker run \
        --publish ${SSH_PORT:-22}:22 \
        --publish ${HTTP_PORT:-80}:80 \
        --publish ${HTTPS_PORT:-443}:443 \
+       --publish 5601:5601 \
        --env SETUP_HOSTNAME="$SETUP_HOSTNAME" \
        --hostname dojo \
        --name pwn.college \