Indexer file: Part 1 (Whole thing is too large for 1 comment):
input {
redis {
host => ["REDACTED"]
port => 6379
key => "logstash"
data_type => "list"
}
}
filter {
ruby {
code => "event['logfilename'] = File.basename(event['path'],File.extname(event['path']))"
}
if ( "apache2_access.log" in [tags] ) {
grok {
match => { "message" => "%{IPORHOST:apache_host} %{COMMONAPACHELOG} - %{QS:referrer} %{QS:unknown} %{QS:agent}" }
match => { "message" => "%{COMMONAPACHELOG}" }
match => { "message" => "(?<NOMATCH>(.|\r|\n)*)" }
}
mutate {
gsub => [
"referrer", "\"", ""
]
}
mutate {
gsub => [
"unknown", "\"", ""
]
}
mutate {
gsub => [
"agent", "\"", ""
]
}
}
if ( "apache2_rewrite.log" in [tags] ) {
grok {
match => { "message" => "%{IPORHOST:remote_host} %{NOTSPACE:remote_logfile} %{NOTSPACE:remote_user} \[%{HTTPDATE}\] \[%{IPORHOST:apache_host}\/sid#(?<sid>(?:((?>(?!\]))).)*)\]\[rid#(?<rid>(?:((?>(?!\/))).)*)\/(?<rid_type>(?:((?>(?!\]))).)*)\] \(%{NUMBER:count}\) (?<msg>(.|\r|\n)*)" }
match => { "message" => "(?<NOMATCH>(.|\r|\n)*)" }
}
}
if ( ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) or ("REDACTED.log" in [tags]) ){
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}\[(?<ExecRequest>[^\]]*)]%{SPACE}%{LOGLEVEL:LogLevel}%{SPACE}(?<JavaClass>[0-9A-Za-z\._]+)-\[(?<Block1>(?:((?>(?!\]-))).)*)\]-\[(?<Block2>(?:((?>(?!\]-))).)*)\]-(?<MainMessage>(.|\r|\n)*)" }
#This is a fallout match. If a log message doesn't match the abover pattern, it will fall into the nomatch bucket
# This will be helpful to identify any pattern matching work that needs to be done in the future
match => { "message" => "(?<NOMATCH>(.|\r|\n)*)" }
}
date {
match => [ "LogTimestamp", "ISO8601" ]
target => "@logtimestamp"
}
if [MainMessage] =~ "\A{(.|\r|\n)*" {
ruby {
code => "event['json'] = event['MainMessage']"
}
}
if ( "" in [json] ) {
json {
source => "json"
}
}
#Removing fields no longer needed
mutate { remove_field => ["json"] }
#Removing REDACTED field
mutate { remove_field => [ "REDACTED" ] }
mutate {
gsub => [
"message", "\"REDACTED\":\"(?:((?>(?!\",))).)*\"", "\"REDACTED\":\"****\"",
"MainMessage", "\"REDACTED\":\"(?:((?>(?!\",))).)*\"", "\"REDACTED\":\"****\""
]
}
if ( [logfilename] in ["REDACTED","REDACTED"] ) {
mutate {
rename => { "Block1" => "SesnId"}
rename => { "Block2" => "ExpId"}
}
}
if ( [logfilename] in ["REDACTED"] ) {
mutate {
rename => { "Block1" => "TranId" }
rename => { "Block2" => "SesnId" }
}
}
#GEO IP
if [RemoteAddress] =~ /(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?|(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9]))/ {
geoip {
source => "RemoteAddress"
database => "/etc/logstash/GeoLiteCity.dat"
}
}
}
