I am seeking help with a Logstash configuration issue. Some of my syslog messages coming into my indexer are in JSON format and some are not. I am solving this using multiple if {} else if {} else {} statements after groking the syslog_message out of the event. (Is there a better way?)
My problem is that this works for some of my events (e.g. player events) but not others (e.g. sysstatf events). What am I doing wrong?
Here are two example input lines:
<190>Jan 28 19:00:32 host2 player-player[1034]: {\"player_message\":\"View finished\",\"player_extra\":{\"slot\":\"com.apps\",\"label\":\"https://s3.amazonaws.com/dev.jpg\"}}
<10>Jan 28 18:55:03 host2 sysstatf.py[14855]: {\"facter\": {\"system_uptime\": {\"seconds\": 617616, \"hours\": 171, \"uptime\": \"7 days\", \"days\": 7}}, \"sysstat\": {\"hosts\": [{\"sysname\": \"Linux\", \"statistics\": [{\"timestamp\": {\"date\": \"2016-01-28\", \"utc\": 1, \"interval\": 120, \"time\": \"18:54:01\"}, \"filesystems\": [{\"MBfsused\": 1483, \"%fsused\": 5.36, \"%Iused\": 4.98, \"Iused\": 89772, \"%ufsused\": 5.36, \"filesystem\": \"/dev/sbd1\", \"MBfsfree\": 26195, \"Ifree\": 1712468}, {\"MBfsused\": 14, \"%fsused\": 21.77, \"%Iused\": 0.0, \"Iused\": 0, \"%ufsused\": 21.77, \"filesystem\": \"/dev/sde1\", \"MBfsfree\": 50, \"Ifree\": 0}], \"cpu-load\": [{\"iowait\": 0.34, \"system\": 3.62, \"idle\": 88.61, \"user\": 7.43, \"steal\": 0.0, \"cpu\": \"all\", \"nice\": 0.0}], \"memory\": {\"memused-percent\": 33.68, \"cached\": 371060, \"memfree\": 1319568, \"inactive\": 172092, \"commit-percent\": 25.93, \"active\": 434424, \"commit\": 515972, \"memused\": 670028, \"buffers\": 157596, \"dirty\": 424}}], \"nodename\": \"host2\", \"file-date\": \"2016-01-28\", \"restarts\": [], \"number-of-cpus\": 4, \"machine\": \"amd64\", \"release\": \"3.4.0-comark\", \"file-utc-time\": \"00:00:01\"}], \"sysdata-version\": 2.19}}
Here is the logstash config:
input {
redis {
port => "6379"
host => "127.0.0.1"
key => "sourceevents"
data_type => "list"
}
}
filter {
if [type] == "syslog" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => { "message" => "<%{POSINT:syslog_pri}>%{COMBOTIMESTAMP:syslog_timestamp} (?:%{SYSLOGHOST:syslog_hostname} )?%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => {
"[@metadata][file_name]" => "source_unknown_syslog.log"
}
}
cidr {
add_tag => [ "index_server" ]
address => [ "%{host}" ]
network => "127.1.0.0/16"
}
cidr {
add_tag => [ "index_network" ]
address => [ "%{host}" ]
network => "127.74.0.0/21"
network => "127.75.0.0/21"
}
cidr {
add_tag => [ "index_xray", "development" ]
address => [ "%{host}" ]
network => "127.16.13.0/24"
}
if "index_server" in [tags] {
mutate {
update => {
"[@metadata][file_name]" => "source_server_syslog.log"
}
}
} else if "index_network" in [tags] {
mutate {
update => {
"[@metadata][file_name]" => "source_network_syslog.log"
}
}
} else if "index_xray" in [tags] {
if [syslog_program] =~ /player.*/ {
json {
add_tag => [ "player_event" ]
source => [ "syslog_message" ]
}
mutate {
update => {
"[@metadata][file_name]" => "source_xray_player.log"
}
}
} else if [syslog_program] == "sysstaf.py" {
json {
add_tag => [ "source_system_statistics", "metrics" ]
source => [ "syslog_message"]
}
mutate {
update => {
"[@metadata][file_name]" => "source_xray_sysstat.log"
}
}
} else {
mutate {
add_tag => "linux"
update => {
"[@metadata][file_name]" => "source_xray_linux.log"
}
}
}
}
}
}
output {
# Order is important: first match takes it
if "_grokparsefailure" not in [tags] and "_grokparsefailure_sysloginput" not in [tags] {
file {
path => "/srv/logstash/%{[@metadata][file_name]}"
}
} else {
file {
path => "/srv/logstash/grokparsefailure.log"
}
}
}
I am hitting the character limit, so I will put the output in another message.