diff --git a/10-rsyslog.conf b/10-rsyslog.conf new file mode 100644 index 0000000..bbe67c5 --- /dev/null +++ b/10-rsyslog.conf @@ -0,0 +1 @@ +*.* @localhost:5140 diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..0cbcdc0 --- /dev/null +++ b/Readme.md @@ -0,0 +1,36 @@ +# Example ELK Box + +This is the vagrant setup for a simple box running the ELK stack. It will make Elasticsearch available on port 9200 and Kibana on port 5601 of the host. + +It setup's up Logstash as a standin for a syslog server, and collects both syslog events and events generated by the [`collectd`](https://www.elastic.co/guide/en/logstash/current/plugins-codecs-collectd.html) input. + +## Howto + +```bash +clone https://github.com/EagerElk/elk-box +cd elk-box +vagrant up +``` + +You can not browse to [localhost:5601](http://localhost:5601) to see your events flowing in. + +## Extra + +Some queries that you can run against Elasticsearch + +```bash +curl 'localhost:9200/_search/' -d '{"aggs":{"events_per":{"date_histogram":{"field":"@timestamp", "interval":"minute"}}}}' +curl 'localhost:9200/_search/?pretty' -d '{"aggs":{"events_per":{"date_histogram":{"field":"@timestamp", "interval":"minute"}}}}' +curl 'localhost:9200/logstash-*/_search/?pretty' -d '{"aggs":{"events_per":{"date_histogram":{"field":"@timestamp", "interval":"hour"}}}}' +curl 'localhost:9200/logstash-*/_search/?pretty' -d '{"size": 0, "aggs":{"events_per":{"date_histogram":{"field":"@timestamp", "interval":"hour"}}}}' +curl 'localhost:9200/logstash-*/_search/?pretty' -d '{"size": 0, "aggs":{"events_per":{"date_histogram":{"field":"@timestamp", "interval":"hour", "min_doc_count": 0\}}}}' + + +curl 'localhost:9200/_search/?q=type:syslog&pretty' +curl 'localhost:9200/_search/?q=type:syslog&pretty' + +curl 'localhost:9200/_search/?q=type:syslog&pretty' +curl 'localhost:9200/_search/?pretty' -d '{"query":{"term":{"type":"syslog"}}}' + +curl 'localhost:9200/_search/?pretty' -d '{"query":{"filtered":{"query":{"term":{"type":"syslog"}},"filter":{"range":{"@timestamp":{"gte":"now-1h"}}}}}}' +``` diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..ccc4d80 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,20 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = 'ubuntu1404' + config.vm.box_url = 'https://vagrantcloud.com/ubuntu/boxes/trusty64/versions/14.04/providers/virtualbox.box' + config.vm.network :private_network, type: :dhcp + config.vm.network :forwarded_port, guest: 9200, host: 9200 + config.vm.network :forwarded_port, guest: 5601, host: 5601 + + config.vm.provider 'virtualbox' do |v| + v.memory = 2048 + end + + config.vm.provision 'ansible' do |ansible| + ansible.playbook = 'setup.yml' + end +end diff --git a/collectd.conf.j2 b/collectd.conf.j2 new file mode 100644 index 0000000..7c006b4 --- /dev/null +++ b/collectd.conf.j2 @@ -0,0 +1,19 @@ +input { + udp { + port => 25826 + buffer_size => 1452 + codec => collectd { } + type => "collectd" + } +} + +output { + if [type] == "collectd" { + elasticsearch { + cluster => "elasticsearch" + host => "localhost" + } + + # stdout { codec => rubydebug } + } +} diff --git a/logstash b/logstash new file mode 100644 index 0000000..f555668 --- /dev/null +++ b/logstash @@ -0,0 +1,40 @@ +############################### +# Default settings for logstash +############################### + +# Override Java location +#JAVACMD=/usr/bin/java + +# Set a home directory +#LS_HOME=/var/lib/logstash + +# Arguments to pass to logstash agent +#LS_OPTS="" + +# Arguments to pass to java +LS_HEAP_SIZE="500m" +#LS_JAVA_OPTS="-Djava.io.tmpdir=$HOME" + +# pidfiles aren't used for upstart; this is for sysv users. +#LS_PIDFILE=/var/run/logstash.pid + +# user id to be invoked as; for upstart: edit /etc/init/logstash.conf +#LS_USER=logstash + +# logstash logging +#LS_LOG_FILE=/var/log/logstash/logstash.log +#LS_USE_GC_LOGGING="true" + +# logstash configuration directory +#LS_CONF_DIR=/etc/logstash/conf.d + +# Open file limit; cannot be overridden in upstart +#LS_OPEN_FILES=16384 + +# Nice level +#LS_NICE=19 + +# If this is set to 1, then when `stop` is called, if the process has +# not exited within a reasonable time, SIGKILL will be sent next. +# The default behavior is to simply log a message "program stop failed; still running" +KILL_ON_STOP_TIMEOUT=1 diff --git a/setup.yml b/setup.yml new file mode 100644 index 0000000..ad403d7 --- /dev/null +++ b/setup.yml @@ -0,0 +1,33 @@ +--- +- hosts: all + remote_user: vagrant + sudo: yes + vars: + elasticsearch_version: 1.7.0 + elasticsearch_cluster_name: elasticsearch + elasticsearch_index_number_of_replicas: 0 + elasticsearch_heap_size: 1g + logstash_conf_files: + - collectd.conf + - syslog.conf + pre_tasks: + - name: Update apt + apt: update_cache=yes cache_valid_time=3600 + roles: + - azavea.kibana + - Stouts.elasticsearch + - geerlingguy.logstash + tasks: + - name: Change Logstash defaults + copy: dest=/etc/default/logstash src=logstash + - name: Configure rsyslog + copy: dest=/etc/rsyslog.d/10-rsyslog.conf src=10-rsyslog.conf + - name: Restart rsyslog + # I'd prefer to use the service module, but it seems to be buggy - http://stackoverflow.com/a/30349016/6681 + # service: name=rsyslog pattern=state=restarted + command: service rsyslog restart + - name: Restart logstash + # I'd prefer to use the service module, but it seems to be buggy - http://stackoverflow.com/a/30349016/6681 + # service: name=rsyslog pattern=state=restarted + command: service logstash restart + diff --git a/syslog.conf.j2 b/syslog.conf.j2 new file mode 100644 index 0000000..486dc26 --- /dev/null +++ b/syslog.conf.j2 @@ -0,0 +1,17 @@ +input { + syslog { + type => "syslog" + port => 5140 + } +} + +output { + if [type] == "syslog" { + elasticsearch { + cluster => "elasticsearch" + host => "localhost" + } + + stdout { codec => rubydebug } + } +}