diff --git a/docs/logit/logit/README.md b/docs/logit/logit/README.md new file mode 100644 index 000000000..a15750e96 --- /dev/null +++ b/docs/logit/logit/README.md @@ -0,0 +1,7 @@ +# Intro + +A collection of config files (mainly logstash) for use on logit.io. + +## Usage + +Currently these are uploaded to Logit via the dashboard. diff --git a/docs/logit/logit/aws_logstash.conf b/docs/logit/logit/aws_logstash.conf new file mode 100644 index 000000000..23d62fedb --- /dev/null +++ b/docs/logit/logit/aws_logstash.conf @@ -0,0 +1,60 @@ +filter { + if [message] =~ "json_log"{ + json { + source => "message" + } + } +else if ([application] == "syslog") { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" + } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } + } + + if [type] == "elb" { + grok { + match => ["message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:loadbalancer} %{IP:client_ip}:%{NUMBER:client_port:int} (?:%{IP:backend_ip}:%{NUMBER:backend_port:int}|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} (?:%{NUMBER:elb_status_code:int}|-) (?:%{NUMBER:backend_status_code:int}|-) %{NUMBER:received_bytes:int} %{NUMBER:sent_bytes:int} \"(?:%{WORD:verb}|-) (?:%{GREEDYDATA:request}|-) (?:HTTP/%{NUMBER:httpversion}|-( )?)\" \"%{DATA:userAgent}\"( %{NOTSPACE:ssl_cipher} %{NOTSPACE:ssl_protocol})?"] + } + + grok { + match => ["request", "%{URIPROTO:http_protocol}"] + } + + if [request] != "-" { + grok { + match => ["request", "(?[^?]*)"] + overwrite => ["request"] + } + } + + geoip { + source => "client_ip" + target => "geoip" + add_tag => ["geoip"] + } + + useragent { + source => "userAgent" + } + + date { + match => ["timestamp", "ISO8601"] + } + } + + if ([application] == 'finder-frontend') { + # https://www.elastic.co/guide/en/logstash/current/plugins-filters-kv.html#plugins-filters-kv-field_split + kv { + source => "request" + field_split => "&? " + prefix => "query." + } + } +} diff --git a/docs/logit/logit/logstash.conf b/docs/logit/logit/logstash.conf new file mode 100644 index 000000000..8e9914940 --- /dev/null +++ b/docs/logit/logit/logstash.conf @@ -0,0 +1,20 @@ +filter { + if [message] =~ "json_log"{ + json { + source => "message" + } + } +else if ([application] == "syslog") { + grok { + match => { + "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" + } + add_field => [ "received_at", "%{@timestamp}" ] + add_field => [ "received_from", "%{host}" ] + } + syslog_pri { } + date { + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } + } +}