diff --git a/.env b/.env index b3cb11f..7308593 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ COMPOSE_PROJECT_NAME=elastic -ELK_VERSION=8.8.0 +ELK_VERSION=8.10.2 #----------- Resources --------------------------# ELASTICSEARCH_HEAP=1024m @@ -15,7 +15,6 @@ KIBANA_HOST=kibana KIBANA_PORT=5601 LOGSTASH_HOST=logstash -LOGSTASH_PORT=8080 APMSERVER_HOST=apm-server APMSERVER_PORT=8200 diff --git a/Makefile b/Makefile index cba1ac3..d2e73dd 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,14 @@ .DEFAULT_GOAL:=help -COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.tools.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml +COMPOSE_ALL_FILES := -f docker-compose.yml -f docker-compose.monitor.yml -f docker-compose.nodes.yml -f docker-compose.logs.yml COMPOSE_MONITORING := -f docker-compose.yml -f docker-compose.monitor.yml COMPOSE_LOGGING := -f docker-compose.yml -f docker-compose.logs.yml -COMPOSE_TOOLS := -f docker-compose.yml -f docker-compose.tools.yml COMPOSE_NODES := -f docker-compose.yml -f docker-compose.nodes.yml ELK_SERVICES := elasticsearch logstash kibana apm-server ELK_LOG_COLLECTION := filebeat ELK_MONITORING := elasticsearch-exporter logstash-exporter filebeat-cluster-logs -ELK_TOOLS := rubban ELK_NODES := elasticsearch-1 elasticsearch-2 -ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING} ${ELK_TOOLS} +ELK_MAIN_SERVICES := ${ELK_SERVICES} ${ELK_MONITORING} ELK_ALL_SERVICES := ${ELK_MAIN_SERVICES} ${ELK_NODES} ${ELK_LOG_COLLECTION} compose_v2_not_supported = $(shell command docker compose 2> /dev/null) @@ -21,7 +19,7 @@ else endif # -------------------------- -.PHONY: setup keystore certs all elk monitoring tools build down stop restart rm logs +.PHONY: setup keystore certs all elk monitoring build down stop restart rm logs keystore: ## Setup Elasticsearch Keystore, by initializing passwords, and add credentials defined in `keystore.sh`. $(DOCKER_COMPOSE_COMMAND) -f docker-compose.setup.yml run --rm keystore @@ -41,17 +39,14 @@ elk: ## Start ELK. up: @make elk - @echo "Visit Kibana: https://localhost:5601" + @echo "Visit Kibana: https://localhost:5601 (user: elastic, password: changeme) [Unless you changed values in .env]" monitoring: ## Start ELK Monitoring. $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_MONITORING} up -d --build ${ELK_MONITORING} -collect-docker-logs: ## Start Filebeat that collects all Host Docker Logs and ship it to ELK +collect-docker-logs: ## Start Filebeat that collects all Host Docker Logs and ship it to ELK $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_LOGGING} up -d --build ${ELK_LOG_COLLECTION} -tools: ## Start ELK Tools (ElastAlert, Curator). - $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_TOOLS} up -d --build ${ELK_TOOLS} - nodes: ## Start Two Extra Elasticsearch Nodes $(DOCKER_COMPOSE_COMMAND) ${COMPOSE_NODES} up -d --build ${ELK_NODES} diff --git a/README.md b/README.md index 926af56..66da61d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@

Configured to be ready to be used for Log, Metrics, APM, Alerting, Machine Learning, and Security (SIEM) usecases.

- Elastic Stack Version 7^^ + Elastic Stack Version 7^^ @@ -36,7 +36,7 @@ Elastic Stack (**ELK**) Docker Composition, preconfigured with **Security**, **M Suitable for Demoing, MVPs and small production deployments. -Stack Version: [8.8.0](https://www.elastic.co/blog/whats-new-elastic-8-8-0) 🎉 - Based on [Official Elastic Docker Images](https://www.docker.elastic.co/) +Stack Version: [8.10.2](https://www.elastic.co/blog/whats-new-elastic-8-10-0) 🎉 - Based on [Official Elastic Docker Images](https://www.docker.elastic.co/) > You can change Elastic Stack version by setting `ELK_VERSION` in `.env` file and rebuild your images. Any version >= 8.0.0 is compatible with this template. ### Main Features 📜 @@ -45,18 +45,18 @@ Stack Version: [8.8.0](https://www.elastic.co/blog/whats-new-elastic-8-8-0) 🎉 - Security Enabled By Default. - Configured to Enable: - Logging & Metrics Ingestion + - Option to collect logs of all Docker Containers running on the host. via `make collect-docker-logs`. - APM - Alerting - Machine Learning - - SIEM + - Anomaly Detection + - SIEM (Security information and event management). - Enabling Trial License - Use Docker-Compose and `.env` to configure your entire stack parameters. - Persist Elasticsearch's Keystore and SSL Certifications. - Self-Monitoring Metrics Enabled. - Prometheus Exporters for Stack Metrics. -- Collect Docker Host Logs to ELK via `make collect-docker-logs`. - Embedded Container Healthchecks for Stack Images. -- [Rubban](https://github.com/sherifabdlnaby/rubban) for Kibana curating tasks. #### More points And comparing Elastdocker and the popular [deviantony/docker-elk](https://github.com/deviantony/docker-elk) @@ -85,7 +85,7 @@ Elastdocker differs from `deviantony/docker-elk` in the following points. - Configuring the Self-Monitoring and the Filebeat agent that ship ELK logs to ELK itself. (as a step to shipping it to a monitoring cluster in the future). -- Configured tools and Prometheus Exporters. +- Configured Prometheus Exporters. - The Makefile that simplifies everything into some simple commands. @@ -135,10 +135,6 @@ Elastdocker differs from `deviantony/docker-elk` in the following points. ```shell $ make monitoring ``` -#### To Start Tools -```shell -$ make tools -``` #### To Ship Docker Container Logs to ELK ```shell $ make collect-docker-logs @@ -172,7 +168,7 @@ $ make prune * Some Configuration are parameterized in the `.env` file. * `ELASTIC_PASSWORD`, user `elastic`'s password (default: `changeme` _pls_). - * `ELK_VERSION` Elastic Stack Version (default: `8.8.0`) + * `ELK_VERSION` Elastic Stack Version (default: `8.10.2`) * `ELASTICSEARCH_HEAP`, how much Elasticsearch allocate from memory (default: 1GB -good for development only-) * `LOGSTASH_HEAP`, how much Logstash allocate from memory. * Other configurations which their such as cluster name, and node name, etc. @@ -180,7 +176,6 @@ $ make prune * Logstash Configuration in `logstash.yml` at `./logstash/config/logstash.yml`. * Logstash Pipeline in `main.conf` at `./logstash/pipeline/main.conf`. * Kibana Configuration in `kibana.yml` at `./kibana/config`. -* Rubban Configuration using Docker-Compose passed Environment Variables. ### Setting Up Keystore diff --git a/docker-compose.nodes.yml b/docker-compose.nodes.yml index e765924..70d737a 100644 --- a/docker-compose.nodes.yml +++ b/docker-compose.nodes.yml @@ -20,6 +20,7 @@ services: ELASTIC_NODE_NAME: ${ELASTIC_NODE_NAME_1} ELASTIC_INIT_MASTER_NODE: ${ELASTIC_INIT_MASTER_NODE} ELASTIC_DISCOVERY_SEEDS: ${ELASTIC_DISCOVERY_SEEDS} + ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT} ES_JAVA_OPTS: -Xmx${ELASTICSEARCH_HEAP} -Xms${ELASTICSEARCH_HEAP} -Des.enforce.bootstrap.checks=true bootstrap.memory_lock: "true" volumes: @@ -56,6 +57,7 @@ services: ELASTIC_NODE_NAME: ${ELASTIC_NODE_NAME_2} ELASTIC_INIT_MASTER_NODE: ${ELASTIC_INIT_MASTER_NODE} ELASTIC_DISCOVERY_SEEDS: ${ELASTIC_DISCOVERY_SEEDS} + ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT} ES_JAVA_OPTS: -Xmx${ELASTICSEARCH_HEAP} -Xms${ELASTICSEARCH_HEAP} -Des.enforce.bootstrap.checks=true bootstrap.memory_lock: "true" volumes: diff --git a/docker-compose.tools.yml b/docker-compose.tools.yml deleted file mode 100644 index ba842c8..0000000 --- a/docker-compose.tools.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3.5' - -services: - rubban: - image: sherifabdlnaby/rubban:latest - restart: unless-stopped - environment: - RUBBAN_KIBANA_HOST: "https://${KIBANA_HOST}:${KIBANA_PORT}" - RUBBAN_KIBANA_USER: ${ELASTIC_USERNAME} - RUBBAN_KIBANA_PASSWORD: ${ELASTIC_PASSWORD} - RUBBAN_REFRESHINDEXPATTERN_ENABLED: 'true' - RUBBAN_REFRESHINDEXPATTERN_SCHEDULE: '*/5 * * * *' - RUBBAN_REFRESHINDEXPATTERN_PATTERNS: '*' - RUBBAN_AUTOINDEXPATTERN_ENABLED: 'true' - RUBBAN_AUTOINDEXPATTERN_SCHEDULE: '*/5 * * * *' - RUBBAN_AUTOINDEXPATTERN_GENERALPATTERNS: '[{"pattern":"filebeat?","timeFieldName":"@timestamp"},{"pattern":"logstash?","timeFieldName":"@timestamp"}]' \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4844380..3d895fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,7 @@ services: ELASTIC_NODE_NAME: ${ELASTIC_NODE_NAME} ELASTIC_INIT_MASTER_NODE: ${ELASTIC_INIT_MASTER_NODE} ELASTIC_DISCOVERY_SEEDS: ${ELASTIC_DISCOVERY_SEEDS} + ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT} ES_JAVA_OPTS: "-Xmx${ELASTICSEARCH_HEAP} -Xms${ELASTICSEARCH_HEAP} -Des.enforce.bootstrap.checks=true -Dlog4j2.formatMsgNoLookups=true" bootstrap.memory_lock: "true" volumes: @@ -63,7 +64,7 @@ services: - source: elasticsearch.key target: /usr/share/elasticsearch/config/certs/elasticsearch.key ports: - - "9200:9200" + - "${ELASTICSEARCH_PORT}:${ELASTICSEARCH_PORT}" - "9300:9300" ulimits: memlock: @@ -73,7 +74,7 @@ services: soft: 200000 hard: 200000 healthcheck: - test: ["CMD", "sh", "-c", "curl -sf --insecure https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@localhost:9200/_cat/health | grep -ioE 'green|yellow' || echo 'not green/yellow cluster status'"] + test: ["CMD", "sh", "-c", "curl -sf --insecure https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@localhost:$ELASTICSEARCH_PORT/_cat/health | grep -ioE 'green|yellow' || echo 'not green/yellow cluster status'"] logstash: image: elastdocker/logstash:${ELK_VERSION} @@ -113,6 +114,7 @@ services: ELASTIC_USERNAME: ${ELASTIC_USERNAME} ELASTIC_PASSWORD: ${ELASTIC_PASSWORD} ELASTICSEARCH_HOST_PORT: https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT} + KIBANA_PORT: ${KIBANA_PORT} env_file: - ./secrets/.env.kibana.token secrets: @@ -123,7 +125,7 @@ services: - source: kibana.key target: /certs/kibana.key ports: - - "5601:5601" + - "${KIBANA_PORT}:${KIBANA_PORT}" apm-server: image: elastdocker/apm-server:${ELK_VERSION} diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml index ac0d2e2..edb0d66 100644 --- a/elasticsearch/config/elasticsearch.yml +++ b/elasticsearch/config/elasticsearch.yml @@ -5,6 +5,7 @@ cluster.name: ${ELASTIC_CLUSTER_NAME} node.name: ${ELASTIC_NODE_NAME} network.host: 0.0.0.0 transport.host: 0.0.0.0 +http.port: ${ELASTICSEARCH_PORT} ## Cluster Settings discovery.seed_hosts: ${ELASTIC_DISCOVERY_SEEDS} diff --git a/elasticsearch/scripts/docker-healthcheck b/elasticsearch/scripts/docker-healthcheck index 89f5820..6f0a223 100644 --- a/elasticsearch/scripts/docker-healthcheck +++ b/elasticsearch/scripts/docker-healthcheck @@ -3,7 +3,7 @@ set -eo pipefail host="$(hostname --ip-address || echo '127.0.0.1')" -if health="$(curl -fsSL "http://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@$host:9200/_cat/health?h=status")"; then +if health="$(curl -fsSL "https://$ELASTIC_USERNAME:$ELASTIC_PASSWORD@$host:$ELASTICSEARCH_PORT/_cat/health?h=status" --insecure")"; then health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ") if [ "$health" = 'green' ] || [ "$health" = "yellow" ]; then exit 0 diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index b2adbf9..0299a99 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -4,6 +4,7 @@ # server.name: kibana server.host: "0.0.0.0" +server.port: ${KIBANA_PORT} # Elasticsearch Connection elasticsearch.hosts: [ "${ELASTICSEARCH_HOST_PORT}" ]