Merge pull request #1429 from alphagov/add-logit-config

Add configuration related to Logit incl. Logstash configuration

docs/logit/logit/README.md

@@ -0,0 +1,7 @@
+# Intro
+
+A collection of config files (mainly logstash) for use on logit.io.
+
+## Usage
+
+Currently these are uploaded to Logit via the dashboard.

docs/logit/logit/aws_logstash.conf

@@ -0,0 +1,60 @@
+filter {
+  if [message] =~ "json_log"{
+    json {
+      source => "message"
+    }
+  }
+else if ([application] == "syslog") {
+    grok {
+      match => {
+        "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"
+      }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    syslog_pri {  }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss"  ]
+    }
+  }
+
+  if [type] == "elb" {
+      grok {
+         match => ["message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:loadbalancer} %{IP:client_ip}:%{NUMBER:client_port:int} (?:%{IP:backend_ip}:%{NUMBER:backend_port:int}|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} (?:%{NUMBER:elb_status_code:int}|-) (?:%{NUMBER:backend_status_code:int}|-) %{NUMBER:received_bytes:int} %{NUMBER:sent_bytes:int} \"(?:%{WORD:verb}|-) (?:%{GREEDYDATA:request}|-) (?:HTTP/%{NUMBER:httpversion}|-( )?)\" \"%{DATA:userAgent}\"( %{NOTSPACE:ssl_cipher} %{NOTSPACE:ssl_protocol})?"]
+      }
+
+      grok {
+         match => ["request", "%{URIPROTO:http_protocol}"]
+      }
+
+      if [request] != "-" {
+         grok {
+            match => ["request", "(?<request>[^?]*)"]
+            overwrite => ["request"]
+         }
+      }
+
+      geoip {
+         source => "client_ip"
+         target => "geoip"
+         add_tag => ["geoip"]
+      }
+
+      useragent {
+         source => "userAgent"
+      }
+
+      date {
+         match => ["timestamp", "ISO8601"]
+      }
+   }
+
+   if ([application] == 'finder-frontend') {
+     # https://www.elastic.co/guide/en/logstash/current/plugins-filters-kv.html#plugins-filters-kv-field_split
+     kv {
+        source => "request"
+        field_split => "&? "
+        prefix => "query."
+     }
+   }
+}

docs/logit/logit/logstash.conf

@@ -0,0 +1,20 @@
+filter {
+  if [message] =~ "json_log"{
+    json {
+      source => "message"
+    }
+  }
+else if ([application] == "syslog") {
+    grok {
+      match => {
+        "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"
+      }
+      add_field => [ "received_at", "%{@timestamp}" ]
+      add_field => [ "received_from", "%{host}" ]
+    }
+    syslog_pri {  }
+    date {
+      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss"  ]
+    }
+  }
+}