openshift.conf

input {
    file {
        type => "openshift-platform"
        start_position => "beginning"
        path => "/var/log/rsyslog/openshift-platform.log"
        tags => [ "openshift", "syslog" ]
    }
    file {
        type => "openshift-broker"
        start_position => "beginning"
        path => "/var/log/rsyslog/openshift-broker.log"
        tags => [ "openshift", "syslog" ]
    }
    file {
        type => "openshift-console"
        start_position => "beginning"
        path => "/var/log/rsyslog/openshift-console.log"
        tags => [ "openshift", "syslog" ]
    }
    file {
        type => "openshift-node-accept"
        start_position => "beginning"
        path => "/var/log/rsyslog/openshift-node-accept.log"
        tags => [ "openshift", "syslog", "system-check" ]
    }
    file {
        type => "oo_idler"
        start_position => "beginning"
        path => "/var/log/rsyslog/oo_idler.log"
        tags => [ "openshift", "syslog", "idler" ]
    }
    file {
        type => "haproxy_ctld"
        start_position => "beginning"
        path => "/var/log/rsyslog/haproxy_ctld.log"
        tags => [ "openshift", "syslog", "haproxy" ]
    }
    file {
        type => "mongodb"
        start_position => "beginning"
        path => "/var/log/rsyslog/mongo*.log"
        tags => [ "openshift", "syslog", "mongodb" ]
    }
    file {
        type => "activemq"
        start_position => "beginning"
        path => "/var/log/rsyslog/activemq.log"
        tags => [ "openshift", "syslog", "activemq" ]
    }
    file {
        type => "sshd"
        start_position => "beginning"
        path => "/var/log/rsyslog/sshd.log"
        tags => [ "sshd", "syslog" ]
    }
}

filter {
    # preprocessing for syslog messages
    if "syslog" in [tags] {
        grok {
            match => { "message" => "%{SYSLOGTIMESTAMP:log_timestamp} %{SYSLOGHOST:hostname} %{SYSLOGPROG}: (%{LOGLEVEL:log_level}|\[\])?\s*%{GREEDYDATA:log_message}" }
        }
        date { match => [ "log_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ] }
    }

    if [type] == "openshift-platform" {
        grok {
            match => { "log_message" => "%{NOTSPACE:platform-message}" }
        }
        if [platform-message] == "openshift-agent:" {
            grok {
                match => { "log_message" => "openshift-agent: request (?<req_state>start|end): action=%{WORD:req_action},? requestid=%{WORD:req_id}, senderid=%{HOSTNAME:req_sender}, (statuscode=%{INT:req_status}, )?data=%{GREEDYDATA:req_data}" }
                add_tag => [ "openshift-agent" ]
            }
        } else if [platform-message] == "type=metric" {
            grok {
                match => { "log_message" => "type=metric appName=%{WORD:app_name} gear=%{WORD:gear_id} app=%{WORD:app_id} ns=%{WORD:app_namespace} %{GREEDYDATA:raw_metrics}" }
                add_tag => [ "metric" ]
            }
            kv {
                source => "raw_metrics"
                prefix => "openshift.metrics.%{app_namespace}.%{app_name}."
            }
        }
    } else if [type] == "openshift-broker" {
        grok {
            match => { "log_message" => "src=%{WORD:broker_src}" }
        }
        if [broker_src] == "useraction" {
            grok {
                match => { "log_message" => "src=useraction RESULT=%{WORD:useraction_result} STATUS=%{WORD:useraction_status} TIMESTAMP=%{WORD:useraction_timestamp} DATE=%{YEAR}-%{MONTHNUM}-%{MONTHDAY} TIME=%{TIME} ACTION=%{WORD:useraction_action} REQ_ID=%{WORD:useraction_requestid} USER_ID=%{WORD:user_id} LOGIN=%{USERNAME:login} %{GREEDYDATA:useraction_details}" }
            }
        } else if [broker_src] == "app" {
            grok {
                match => { "log_message" => "src=app %{GREEDYDATA:broker_app_message}" }
            }
        } else if [broker_src] == "usage" {
            grok {
                match => { "log_message" => "src=usage User ID: %{WORD:user_id} Event: %{WORD:usage_event} Gear: %{WORD:gear_id} UsageType: %{WORD:usage_type} GearSize: %{WORD:gear_size}" }
            }
        } else if [broker_src] == "osemetric" {
            grok {
                match => { "log_message" => "src=osemetric scope=%{WORD:metric_scope} component=%{WORD:metric_component} name=%{WORD:metric_name} %{GREEDYDATA:raw_metrics}" }
                add_tag => [ "ose-metric" ]
            }
            kv {
                source => "raw_metrics"
                prefix => "openshift.ose-metrics.%{metric_scope}.%{metric_component}.%{metric_name}."
            }
        }
    } else if [type] == "openshift-console" {
        grok {
            match => { "log_message" => "src=app %{GREEDYDATA:console_app_message}" }
            tag_on_failure => []
        }
    } else if [type] == "openshift-frontend" {
        mutate {
          add_tag => [ "openshift-frontend" ]
        }
    } else if [type] == "openshift-node-accept" {
        grok {
            match => { "log_message" => "%{WORD:node_status}: %{GREEDYDATA:}" }
        }
        if [node_status] == "FAIL" {
            mutate {
                add_field => { "log_level" => "ERROR" }
                remove_field => [ "node_status" ]
                add_tag => "node_error"
            }
        } else {
            drop { }
        }
    } else if [type] == "oo_idler" {
        grok {
            match => { "log_message" => "Idled: %{WORD:gear_id}" }
        }
    } else if [type] == "haproxy_ctld" {
        #TODO
    } else if [type] == "mongodb" {
        #TODO
    } else if [type] == "activemq" {
        #TODO
    } else if [type] == "sshd" {
        # Example: detect failed logins and invalid users
        grok {
            match => { "log_message" => "Failed password for invalid user %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2" }
            add_tag => "ssh_invalid_user"
            tag_on_failure => []
        }
        grok {
            match => { "log_message" => "Failed password for %{USERNAME:username} from %{IP:src_ip} port %{BASE10NUM:port} ssh2" }
            add_tag => "ssh_failed_password"
            tag_on_failure => []
        }
    } else {
        drop {  }
    }
}


output {
    stdout {
        codec => "rubydebug"
    }
    # push metrics to graphite
    # push others to elasticsearch
    # we use a separate tag for platform (non-app) metrics
    if "ose-metric" in [tags] {
        graphite {
            host => "graphite.example.com"
            include_metrics => "openshift.ose-metrics.*"
            fields_are_metrics => true
        }
    } else if "metric" in [tags] {
        graphite {
            host => "graphite.example.com"
            include_metrics => "openshift.metrics.*"
            fields_are_metrics => true
        }
    } else {
        elasticsearch {
            protocol => "http"
            cluster => "openshift"
        }
    }
}