The only other json calls that use message should never have an empty line. They are a { "foo": "bar", ... , "bar": "foo" } object per line. So the filter looks like:
json { source => "message" }
Well, I was trying to avoid posting a bunch of code like this, but here is the full config for the only json logs I have:
if [source] == "/var/log/apache2/access_json.log" {
json {
source => "message"
}
mutate {
gsub => [
"forwardedip","127.0.0.1","",
"forwardedip",",",""
]
strip => ["forwardedip"]
}
mutate { convert => { "bytes" => "integer" } }
if [forwardedip] !~ /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/ {
if (([clientip] == "-") or ([clientip] == [proxyip])) and ([forwardedip] != "-") {
mutate {
replace => { "clientip" => "%{[forwardedip]}" }
}
}
}
if [proxyip] == [clientip] { mutate { replace => { "proxyip" => "-" } } }
if [clientip] !~ /(^127\.0\.0\.1)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/ {
geoip {
source => "clientip"
}
}
}
My apache log format:
LogFormat "{\"timestamp\": \"%t\",\"forwardedip\": \"%{X-Forwarded-For}i\",\"clientip\": \"%a\",\"proxyip\": \"%{c}a\",\"serverip\": \"%A\",\"servername\": \"%v\",\"request_log_id\": \"%L\",\"port\": \"%p\",\"ident\": \"%l\",\"auth\": \"%u\",\"verb\": \"%m\",\"request\": \"%U\",\"query\": \"%q\",\"httpversion\": \"%H\",\"response\": \"%>s\",\"bytes\": \"%b\",\"referrer\": \"%{Referer}i\",\"agent\": \"%{User-Agent}i\"}" access_json
if [type] == "ldapwrangler" {
json {
source => "message"
}
mutate {
convert => { "timestamp" => "integer" }
}
date {
match => [ "timestamp", "UNIX" ]
target => "@timestamp"
}
}
ldapwrangler logs are a php array encoded to json and written to a text file.
if [type] == "gitlab_production" {
grok {
match => ["message", "Started\s%{WORD:verb}\s\"%{NOTSPACE:request}\"\sfor\s%{IPORHOST:clientip}\s(at)\s.*(?<timestamp>%{TIMESTAMP_ISO8601}\s[-+]{1}\d{4})((?<main_message>.*)Parameters:\s(?<project_parameters>\{.*\})(?<main_message_two>.*\n)*(Completed?\s?%{NUMBER:response}(?<response_message>.*))?)?"]
add_field => {"combinestuff" => "blah" }
}
mutate {
gsub => ["project_parameters", "=>",":"]
}
if [project_parameters]{
json {
source => "project_parameters"
target => "project_parameters"
}}
if [main_message_two]{
mutate {
update => {"combinestuff" => "%{main_message} (project_parameters) %{main_message_two}"}
remove_field => ["main_message_two", "main_message"]
}
}
if [response_message] and [response] {
mutate {
update => {"combinestuff" => "%{combinestuff} Response %{response} %{response_message}"}
remove_field => ["response_message"]
}
}
if [combinestuff] != "blah"{
mutate {
update => {"message" => "%{combinestuff}"}
remove_field => ["combinestuff"]
}
}
if [combinestuff] == "blah"{
mutate {
remove_field => ["combinestuff"]
}
}
date {
match => [ "timestamp", "YYYY-MM-dd HH:mm:ss Z" ]
target => "@timestamp"
}
}
See my previous post about the gitlab production logs.