proxy-iwsva.conf

# vim: syntax=python

# Please check https://github.com/cvandeplas/ELK-forensics for more information.
# Created by Christophe Vandeplas <christophe@vandeplas.com>

input {
  syslog {
    type => "proxy_fwd_iwsva"
    port => 8514
  }
}

filter {
  if [type] == "proxy_fwd_iwsva" {
    grok {
      match => ["message", "<(?<timestamp>.+)> \[(?<event_type>.+)\] Access tracking log (?<data_kv>.*)"]
      overwrite => ["timestamp"]
    }
    if ("_grokparsefailure" in [tags]) {
      drop {}
    }

    kv {   # split key-value pairs of the line
      source => "data_kv"
      field_split => ","
      # TODO - remove this later: work around a bug in the kv structure of the log with some URLs
      include_keys => ["tk_category", "tk_category_type", "tk_client_ip", "tk_date_field", "tk_domain", "tk_file_name", "tk_mime_content", "tk_operation", "tk_path", "tk_protocol", "tk_server", "tk_server_ip", "tk_size", "tk_uid", "tk_url", "tk_username"]
    }
    date {
      match => ["timestamp", "EEE, dd MMM YYYY HH:mm:ss',CEST'"]
      timezone => ["Europe/Brussels"]
    }
    geoip { 
      source => "tk_server_ip"
    }
    # convert the category number to text
    translate {
      field => "tk_category"
      destination => "category"
      override => true
      dictionary => [   "1", "Adult/Mature Content",
                        "3", "Pornography",
                        "4", "Sex Education",
                        "5", "Intimate Apparel/Swimsuit",
                        "6", "Nudity",
                        "8", "Alcohol/Tobacco",
                        "9", "Illegal/Questionable",
                        "10", "Tasteless",
                        "11", "Gambling",
                        "14", "Violence/Hate/Racism",
                        "15", "Weapons",
                        "16", "Abortion",
                        "18", "Recreation/Hobbies",
                        "19", "Arts",
                        "20", "Entertainment",
                        "21", "Business/Economy",
                        "22", "Cult/Occult",
                        "23", "Internet Radio and TV",
                        "24", "Internet Telephony",
                        "25", "Illegal Drugs",
                        "26", "Marijuana",
                        "27", "Education",
                        "29", "Cultural Institutions",
                        "30", "Activist Groups",
                        "31", "Financial Services",
                        "32", "Brokerage/Trading",
                        "33", "Games",
                        "34", "Government/Legal",
                        "35", "Military",
                        "36", "Politics",
                        "37", "Health",
                        "38", "Computers/Internet",
                        "39", "Proxy Avoidance",
                        "40", "Search Engines/Portals",
                        "41", "Internet Infrastructure",
                        "42", "Blogs/Web Communications",
                        "43", "Photo Searches",
                        "44", "Alternative Journals",
                        "45", "Job Search/Careers",
                        "46", "News/Media",
                        "47", "Personals/Dating",
                        "48", "Translators / Cached Pages",
                        "49", "Reference",
                        "50", "Social Networking",
                        "51", "Chat/Instant Messaging",
                        "52", "Email",
                        "53", "Newsgroups",
                        "54", "Religion",
                        "55", "Personal Sites",
                        "56", "Personal Network Storage/File Download Server",
                        "57", "Peer-to-peer",
                        "58", "Shopping",
                        "59", "Auctions",
                        "60", "Real Estate",
                        "61", "Society/Lifestyle",
                        "62", "Gay/Lesbian",
                        "63", "Gun Clubs/Hunting",
                        "64", "Restaurants/Food",
                        "65", "Sports",
                        "66", "Travel",
                        "67", "Vehicles",
                        "68", "Humor",
                        "69", "Streaming Media/MP3",
                        "70", "Ringtones/Mobile Phone Downloads",
                        "71", "Software Downloads",
                        "72", "Pay to Surf",
                        "73", "Potentially Malicious Software",
                        "74", "Spyware",
                        "75", "Phishing",
                        "76", "Spam",
                        "77", "Adware",
                        "78", "Malware Accomplice",
                        "79", "Disease Vector",
                        "80", "Cookies",
                        "81", "Dialers",
                        "82", "Hacking",
                        "83", "Joke Program",
                        "84", "Password Cracking",
                        "85", "Remote Access Program",
                        "86", "Made for AdSense",
                        "87", "For Kids",
                        "88", "Web Advertisement",
                        "89", "Web Hosting",
                        "90", "Untested"
                        ]
    }
    mutate {
      convert => ["tk_size", "integer"]
      remove_field => ["host", "priority", "severity", "facility", "facility_label", "severity_label",  "data_kv", "message", "tk_server", "program", "logsource", "event_type" ]
    }
  }
}

output {
  if [type] == "proxy_fwd_iwsva" {
    elasticsearch {
      index => "logstash-%{[type]}-%{+YYYY.MM.dd}"
      host => "localhost"
    }
  }
}