diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4dcc3fe..d2b9675 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -15,11 +15,3 @@ jobs: - uses: docker/setup-buildx-action@v3 - run: docker-compose -f elastic-stack/docker-compose.yml build - run: docker-compose -f serilog/docker-compose.yml build - - update_readme: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: ./scripts/update-readme.sh diff --git a/README.md b/README.md index 80cda70..adcc47d 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,16 @@ + # Serilog and the Elastic Stack + ## Table of contents - [Introduction](#introduction) - [What you will end up with](#what-you-will-end-up-with) - [Requirements](#requirements) -- [Usage on Windows](#usage-on-windows) -- [Usage on Linux and macOS](#usage-on-linux-and-macos) +- [Usage](#usage) + - [Bringing up Elastic Stack](#bringing-up-elastic-stack) + - [Publishing log events using Serilog](#publishing-log-events-using-serilog) + - [Using Kibana to render the log events](#using-kibana-to-render-the-log-events) - [Credit](#credit) --- @@ -25,29 +29,19 @@ With a running Elastic Stack and Serilog producing log events you are now ready - [Docker](https://www.docker.com/community-edition#/download) -## Usage on Windows +## Usage ### Bringing up Elastic Stack Start the stack using `docker`: ```posh -PS> cd .\elastic-stack\ -PS> docker compose up -``` +cd .\elastic-stack\ -If this is the first time the stack is started, you'll have to create a Logstash index pattern. Give the stack some time to initialize and then run the following commands in PowerShell: +# This command is only necessary the first time the stack is started +docker compose up setup -```posh -PS> $Headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]" -PS> $Headers.Add("Content-Type", "application/json") -PS> $Headers.Add("kbn-version", "7.17.0") -PS> $Auth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("elastic:changeme")) -PS> $Headers.Add("Authorization", "Basic {0}" -f $Auth) -PS> Invoke-RestMethod "http://localhost:5601/api/saved_objects/index-pattern" ` - -Method Post ` - -Headers $Headers ` - -Body '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' +docker compose up ``` ### Publishing log events using Serilog @@ -55,8 +49,8 @@ PS> Invoke-RestMethod "http://localhost:5601/api/saved_objects/index-pattern" ` Run the following commands to publish log events to Logstash using Serilog: ```posh -PS> cd .\serilog\ -PS> docker compose up +cd .\serilog\ +docker compose up ``` If you decide to run the application outside of Docker in your terminal, don't forget to change the request URI to `http://localhost:31311`. More information can be found in `.\serilog\Program.cs`. @@ -65,42 +59,6 @@ If you decide to run the application outside of Docker in your terminal, don't f Access the Kibana web UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser, and when prompted enter username `elastic` and password `changeme`. -## Usage on Linux and macOS - -### Bringing up Elastic Stack - -Start the stack using `docker`: - -```bash -$ cd elastic-stack/ -$ docker compose up -``` - -If this is the first time the stack is started, you'll have to create a Logstash index pattern. Give the stack some time to initialize and then run the following commands: - -```bash -$ curl -XPOST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \ - -H 'Content-Type: application/json' \ - -H 'kbn-version: 7.17.0' \ - -u elastic:changeme \ - -d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}' -``` - -### Publishing log events using Serilog - -Run the following commands to publish log events to Logstash using Serilog: - -```bash -$ cd serilog/ -$ docker compose up -``` - -If you decide to run the application outside of Docker in your terminal, don't forget to change the request URI to `http://localhost:31311`. More information can be found in `./serilog/Program.cs`. - -### Using Kibana to render the log events - -Access the Kibana web UI by hitting [http://localhost:5601](http://localhost:5601) with a web browser, and when prompted enter username `elastic` and password `changeme`. - ## Credit The `elastic-stack` directory is a clone of [docker-elk](https://github.com/deviantony/docker-elk) with minor modifications. Credit to [deviantony](https://github.com/deviantony) for publishing the Elastic Stack boilerplate. diff --git a/elastic-stack/.env b/elastic-stack/.env index e40c94c..b178e6a 100644 --- a/elastic-stack/.env +++ b/elastic-stack/.env @@ -1 +1,22 @@ -ELK_VERSION=7.17.0 +ELASTIC_VERSION=8.13.2 + +## Passwords for stack users +# + +# User 'elastic' (built-in) +# +# Superuser role, full access to cluster management and data indices. +# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html +ELASTIC_PASSWORD='changeme' + +# User 'logstash_internal' (custom) +# +# The user Logstash uses to connect and send data to Elasticsearch. +# https://www.elastic.co/guide/en/logstash/current/ls-security.html +LOGSTASH_INTERNAL_PASSWORD='changeme' + +# User 'kibana_system' (built-in) +# +# The user Kibana uses to connect and communicate with Elasticsearch. +# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html +KIBANA_SYSTEM_PASSWORD='changeme' diff --git a/elastic-stack/docker-compose.yml b/elastic-stack/docker-compose.yml index ba5932d..65b6416 100644 --- a/elastic-stack/docker-compose.yml +++ b/elastic-stack/docker-compose.yml @@ -1,57 +1,100 @@ -version: '3.2' - services: + # The 'setup' service runs a one-off script which initializes users inside + # Elasticsearch — such as 'logstash_internal' and 'kibana_system' — with the + # values of the passwords defined in the '.env' file. It also creates the + # roles required by some of these users. + # + # This task only needs to be performed once, during the *initial* startup of + # the stack. Any subsequent run will reset the passwords of existing users to + # the values defined inside the '.env' file, and the built-in roles to their + # default permissions. + # + # By default, it is excluded from the services started by 'docker compose up' + # due to the non-default profile it belongs to. To run it, either provide the + # '--profile=setup' CLI flag to Compose commands, or "up" the service by name + # such as 'docker compose up setup'. + setup: + profiles: + - setup + build: + context: setup/ + args: + ELASTIC_VERSION: ${ELASTIC_VERSION} + init: true + volumes: + - ./setup/entrypoint.sh:/entrypoint.sh:ro,Z + - ./setup/lib.sh:/lib.sh:ro,Z + - ./setup/roles:/roles:ro,Z + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} + LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-} + KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} + networks: + - elk + depends_on: + - elasticsearch + elasticsearch: build: context: elasticsearch/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z - - elasticsearch:/usr/share/elasticsearch/data:z + - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,Z + - elasticsearch:/usr/share/elasticsearch/data:Z ports: - - "9200:9200" - - "9300:9300" + - 9200:9200 + - 9300:9300 environment: - ES_JAVA_OPTS: "-Xmx256m -Xms256m" - ELASTIC_PASSWORD: changeme + node.name: elasticsearch + ES_JAVA_OPTS: -Xms512m -Xmx512m + # Bootstrap password. + # Used to initialize the keystore during the initial startup of + # Elasticsearch. Ignored on subsequent runs. + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} # Use single node discovery in order to disable production mode and avoid bootstrap checks. # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html discovery.type: single-node networks: - elk + restart: unless-stopped logstash: build: context: logstash/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,z - - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,z + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z + - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z ports: - - "9600:9600" - - "31311:31311" + - 9600:9600 + - 31311:31311 environment: - LS_JAVA_OPTS: "-Xmx256m -Xms256m" + LS_JAVA_OPTS: -Xms256m -Xmx256m + LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-} networks: - elk depends_on: - elasticsearch + restart: unless-stopped kibana: build: context: kibana/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,z + - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z ports: - - "5601:5601" + - 5601:5601 + environment: + KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} networks: - elk depends_on: - elasticsearch + restart: unless-stopped networks: elk: diff --git a/elastic-stack/elasticsearch/.dockerignore b/elastic-stack/elasticsearch/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/elastic-stack/elasticsearch/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/elastic-stack/elasticsearch/Dockerfile b/elastic-stack/elasticsearch/Dockerfile index 3928544..22528c6 100644 --- a/elastic-stack/elasticsearch/Dockerfile +++ b/elastic-stack/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} # Add your elasticsearch plugins setup here # Example: RUN elasticsearch-plugin install analysis-icu diff --git a/elastic-stack/elasticsearch/config/elasticsearch.yml b/elastic-stack/elasticsearch/config/elasticsearch.yml index 86822dd..d66f071 100644 --- a/elastic-stack/elasticsearch/config/elasticsearch.yml +++ b/elastic-stack/elasticsearch/config/elasticsearch.yml @@ -1,13 +1,12 @@ --- ## Default Elasticsearch configuration from Elasticsearch base image. -## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml +## https://github.com/elastic/elasticsearch/blob/main/distribution/docker/src/docker/config/elasticsearch.yml # -cluster.name: "docker-cluster" +cluster.name: docker-cluster network.host: 0.0.0.0 ## X-Pack settings -## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html # xpack.license.self_generated.type: trial xpack.security.enabled: true -xpack.monitoring.collection.enabled: true diff --git a/elastic-stack/kibana/.dockerignore b/elastic-stack/kibana/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/elastic-stack/kibana/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/elastic-stack/kibana/Dockerfile b/elastic-stack/kibana/Dockerfile index 2fb3659..9a075be 100644 --- a/elastic-stack/kibana/Dockerfile +++ b/elastic-stack/kibana/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} +FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION} # Add your kibana plugins setup here # Example: RUN kibana-plugin install diff --git a/elastic-stack/kibana/config/kibana.yml b/elastic-stack/kibana/config/kibana.yml index 0e1dc60..ef3f024 100644 --- a/elastic-stack/kibana/config/kibana.yml +++ b/elastic-stack/kibana/config/kibana.yml @@ -1,13 +1,99 @@ --- ## Default Kibana configuration from Kibana base image. -## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts +## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts # server.name: kibana server.host: 0.0.0.0 -elasticsearch.hosts: [ "http://elasticsearch:9200" ] +elasticsearch.hosts: [ http://elasticsearch:9200 ] + monitoring.ui.container.elasticsearch.enabled: true +monitoring.ui.container.logstash.enabled: true ## X-Pack security credentials # -elasticsearch.username: elastic -elasticsearch.password: changeme +elasticsearch.username: kibana_system +elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD} + +## Encryption keys (optional but highly recommended) +## +## Generate with either +## $ docker container run --rm docker.elastic.co/kibana/kibana:8.6.2 bin/kibana-encryption-keys generate +## $ openssl rand -hex 32 +## +## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html +## https://www.elastic.co/guide/en/kibana/current/kibana-encryption-keys.html +# +#xpack.security.encryptionKey: +#xpack.encryptedSavedObjects.encryptionKey: +#xpack.reporting.encryptionKey: + +## Fleet +## https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html +# +xpack.fleet.agents.fleet_server.hosts: [ http://fleet-server:8220 ] + +xpack.fleet.outputs: + - id: fleet-default-output + name: default + type: elasticsearch + hosts: [ http://elasticsearch:9200 ] + is_default: true + is_default_monitoring: true + +xpack.fleet.packages: + - name: fleet_server + version: latest + - name: system + version: latest + - name: elastic_agent + version: latest + - name: docker + version: latest + - name: apm + version: latest + +xpack.fleet.agentPolicies: + - name: Fleet Server Policy + id: fleet-server-policy + description: Static agent policy for Fleet Server + monitoring_enabled: + - logs + - metrics + package_policies: + - name: fleet_server-1 + package: + name: fleet_server + - name: system-1 + package: + name: system + - name: elastic_agent-1 + package: + name: elastic_agent + - name: docker-1 + package: + name: docker + - name: Agent Policy APM Server + id: agent-policy-apm-server + description: Static agent policy for the APM Server integration + monitoring_enabled: + - logs + - metrics + package_policies: + - name: system-1 + package: + name: system + - name: elastic_agent-1 + package: + name: elastic_agent + - name: apm-1 + package: + name: apm + # See the APM package manifest for a list of possible inputs. + # https://github.com/elastic/apm-server/blob/v8.5.0/apmpackage/apm/manifest.yml#L41-L168 + inputs: + - type: apm + vars: + - name: host + value: 0.0.0.0:8200 + - name: url + value: http://apm-server:8200 diff --git a/elastic-stack/logstash/.dockerignore b/elastic-stack/logstash/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/elastic-stack/logstash/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/elastic-stack/logstash/Dockerfile b/elastic-stack/logstash/Dockerfile index 6a444e7..bde5808 100644 --- a/elastic-stack/logstash/Dockerfile +++ b/elastic-stack/logstash/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} +FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION} # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json diff --git a/elastic-stack/logstash/config/logstash.yml b/elastic-stack/logstash/config/logstash.yml index a48c35f..a81b89b 100644 --- a/elastic-stack/logstash/config/logstash.yml +++ b/elastic-stack/logstash/config/logstash.yml @@ -1,12 +1,7 @@ --- ## Default Logstash configuration from Logstash base image. -## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml +## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml # -http.host: "0.0.0.0" -xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] +http.host: 0.0.0.0 -## X-Pack security credentials -# -xpack.monitoring.enabled: true -xpack.monitoring.elasticsearch.username: elastic -xpack.monitoring.elasticsearch.password: changeme +node.name: logstash diff --git a/elastic-stack/logstash/pipeline/logstash.conf b/elastic-stack/logstash/pipeline/logstash.conf index 567e542..6b4369c 100644 --- a/elastic-stack/logstash/pipeline/logstash.conf +++ b/elastic-stack/logstash/pipeline/logstash.conf @@ -10,8 +10,7 @@ input { output { elasticsearch { hosts => "elasticsearch:9200" - user => "elastic" - password => "changeme" - ecs_compatibility => disabled + user => "logstash_internal" + password => "${LOGSTASH_INTERNAL_PASSWORD}" } } diff --git a/elastic-stack/setup/.dockerignore b/elastic-stack/setup/.dockerignore new file mode 100644 index 0000000..c5dd1c8 --- /dev/null +++ b/elastic-stack/setup/.dockerignore @@ -0,0 +1,9 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store + +# Ignore Git files +.gitignore diff --git a/elastic-stack/setup/Dockerfile b/elastic-stack/setup/Dockerfile new file mode 100644 index 0000000..1cb7538 --- /dev/null +++ b/elastic-stack/setup/Dockerfile @@ -0,0 +1,6 @@ +ARG ELASTIC_VERSION + +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/elastic-stack/setup/entrypoint.sh b/elastic-stack/setup/entrypoint.sh new file mode 100755 index 0000000..9f10754 --- /dev/null +++ b/elastic-stack/setup/entrypoint.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source "${BASH_SOURCE[0]%/*}"/lib.sh + + +# -------------------------------------------------------- +# Users declarations + +declare -A users_passwords +users_passwords=( + [logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}" + [kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}" +) + +declare -A users_roles +users_roles=( + [logstash_internal]='logstash_writer' +) + +# -------------------------------------------------------- +# Roles declarations + +declare -A roles_files +roles_files=( + [logstash_writer]='logstash_writer.json' +) + +# -------------------------------------------------------- + + +log 'Waiting for availability of Elasticsearch. This can take several minutes.' + +declare -i exit_code=0 +wait_for_elasticsearch || exit_code=$? + +if ((exit_code)); then + case $exit_code in + 6) + suberr 'Could not resolve host. Is Elasticsearch running?' + ;; + 7) + suberr 'Failed to connect to host. Is Elasticsearch healthy?' + ;; + 28) + suberr 'Timeout connecting to host. Is Elasticsearch healthy?' + ;; + *) + suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}" + ;; + esac + + exit $exit_code +fi + +sublog 'Elasticsearch is running' + +log 'Waiting for initialization of built-in users' + +wait_for_builtin_users || exit_code=$? + +if ((exit_code)); then + suberr 'Timed out waiting for condition' + exit $exit_code +fi + +sublog 'Built-in users were initialized' + +for role in "${!roles_files[@]}"; do + log "Role '$role'" + + declare body_file + body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}" + if [[ ! -f "${body_file:-}" ]]; then + sublog "No role body found at '${body_file}', skipping" + continue + fi + + sublog 'Creating/updating' + ensure_role "$role" "$(<"${body_file}")" +done + +for user in "${!users_passwords[@]}"; do + log "User '$user'" + if [[ -z "${users_passwords[$user]:-}" ]]; then + sublog 'No password defined, skipping' + continue + fi + + declare -i user_exists=0 + user_exists="$(check_user_exists "$user")" + + if ((user_exists)); then + sublog 'User exists, setting password' + set_user_password "$user" "${users_passwords[$user]}" + else + if [[ -z "${users_roles[$user]:-}" ]]; then + suberr ' No role defined, skipping creation' + continue + fi + + sublog 'User does not exist, creating' + create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}" + fi +done diff --git a/elastic-stack/setup/lib.sh b/elastic-stack/setup/lib.sh new file mode 100755 index 0000000..7e635c6 --- /dev/null +++ b/elastic-stack/setup/lib.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash + +# Log a message. +function log { + echo "[+] $1" +} + +# Log a message at a sub-level. +function sublog { + echo " ⠿ $1" +} + +# Log an error. +function err { + echo "[x] $1" >&2 +} + +# Log an error at a sub-level. +function suberr { + echo " ⠍ $1" >&2 +} + +# Poll the 'elasticsearch' service until it responds with HTTP code 200. +function wait_for_elasticsearch { + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + # retry for max 300s (60*5s) + for _ in $(seq 1 60); do + local -i exit_code=0 + output="$(curl "${args[@]}")" || exit_code=$? + + if ((exit_code)); then + result=$exit_code + fi + + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + break + fi + + sleep 5 + done + + if ((result)) && [[ "${output: -3}" -ne 000 ]]; then + echo -e "\n${output::-3}" + fi + + return $result +} + +# Poll the Elasticsearch users API until it returns users. +function wait_for_builtin_users { + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + + local line + local -i exit_code + local -i num_users + + # retry for max 30s (30*1s) + for _ in $(seq 1 30); do + num_users=0 + + # read exits with a non-zero code if the last read input doesn't end + # with a newline character. The printf without newline that follows the + # curl command ensures that the final input not only contains curl's + # exit code, but causes read to fail so we can capture the return value. + # Ref. https://unix.stackexchange.com/a/176703/152409 + while IFS= read -r line || ! exit_code="$line"; do + if [[ "$line" =~ _reserved.+true ]]; then + (( num_users++ )) + fi + done < <(curl "${args[@]}"; printf '%s' "$?") + + if ((exit_code)); then + result=$exit_code + fi + + # we expect more than just the 'elastic' user in the result + if (( num_users > 1 )); then + result=0 + break + fi + + sleep 1 + done + + return $result +} + +# Verify that the given Elasticsearch user exists. +function check_user_exists { + local username=$1 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local -i exists=0 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then + result=0 + fi + if [[ "${output: -3}" -eq 200 ]]; then + exists=1 + fi + + if ((result)); then + echo -e "\n${output::-3}" + else + echo "$exists" + fi + + return $result +} + +# Set password of a given Elasticsearch user. +function set_user_password { + local username=$1 + local password=$2 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}/_password" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "{\"password\" : \"${password}\"}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} + +# Create the given Elasticsearch user. +function create_user { + local username=$1 + local password=$2 + local role=$3 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} + +# Ensure that the given Elasticsearch role is up-to-date, create it if required. +function ensure_role { + local name=$1 + local body=$2 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/role/${name}" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "$body" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} diff --git a/elastic-stack/setup/roles/logstash_writer.json b/elastic-stack/setup/roles/logstash_writer.json new file mode 100644 index 0000000..b43861f --- /dev/null +++ b/elastic-stack/setup/roles/logstash_writer.json @@ -0,0 +1,33 @@ +{ + "cluster": [ + "manage_index_templates", + "monitor", + "manage_ilm" + ], + "indices": [ + { + "names": [ + "logs-generic-default", + "logstash-*", + "ecs-logstash-*" + ], + "privileges": [ + "write", + "create", + "create_index", + "manage", + "manage_ilm" + ] + }, + { + "names": [ + "logstash", + "ecs-logstash" + ], + "privileges": [ + "write", + "manage" + ] + } + ] +} diff --git a/scripts/update-readme.sh b/scripts/update-readme.sh deleted file mode 100755 index 88c6be7..0000000 --- a/scripts/update-readme.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Setup git -git config user.name FantasticFiasco -git config user.email mattias@kindb.org - -# Update Elastic Stack version in README -DOCKERFILE_ELASTIC_VERSION=`grep -oP '\d+\.\d+\.\d+' ./elastic-stack/.env` -sed -i -E "s/\"kbn-version\", \"[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\"/\"kbn-version\", \"${DOCKERFILE_ELASTIC_VERSION}\"/g" ./README.md -sed -i -E "s/'kbn-version: [[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'/\'kbn-version: ${DOCKERFILE_ELASTIC_VERSION}\'/g" ./README.md -git add ./README.md -git diff-index --cached --quiet HEAD || git commit -m "docs(readme): update elastic version" - -# Push git changes -git push diff --git a/serilog/docker-compose.yml b/serilog/docker-compose.yml index b8896ee..e441a58 100644 --- a/serilog/docker-compose.yml +++ b/serilog/docker-compose.yml @@ -1,5 +1,3 @@ -version: '2' - services: serilog-example: build: @@ -7,6 +5,6 @@ services: networks: - elastic-stack_elk -networks: +networks: elastic-stack_elk: - external: true \ No newline at end of file + external: true