Hi,
I have a problem with logstash, when i add a grok filter, logstash seem to stop sending data to ES or kibana.
My data come by udp protocol on port 514
here is my filter conf file (the ELK system is working fine at this point)
filter {
# if [type] == "syslog" {
# syslog_pri { }
# date {
# match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
# match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
# }
# }
####FILTER ESX
if [host] == "IPESX1" or [host] == "IPESX2" {
grok{
break_on_match => true
match => [
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}:\s%{WORD:esx_syslog_level}\s%{SYSLOGPROG}\[%{DATA:esx_thread_id}\]\s\[%{DATA} sub=%{DATA:esx_sub} opID=%{DATA:esx_opID}\]\s\[%{DATA:esx_msg_service_info}\]\s%{GREEDYDATA:esx_msg}",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}:\s%{WORD:esx_syslog_level}\s%{SYSLOGPROG}\[%{DATA:esx_thread_id}\]\s\[%{DATA} sub=%{DATA:esx_sub}\]\s%{DATA:esx_msg_service_info}\s:%{GREEDYDATA:esx_msg}",{SYSLOGPROG}\[%{DATA:esx_thread_id}\]\s\[%{DATA} sub=%{DATA:esx_sub}\]\s%{DATA:esx_msg_service_info}:%{GREEDYDATA:esx_msg}",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}:\s%{WORD:esx_syslog_level}\s%{GREEDYDATA:esx_msg}",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}\[%{DATA:esx_thread_id}\]:%{GREEDYDATA:esx_msg}",
"message", "<%{POSINT:syslog_pri}>-->\s\[%{DATA:msg}\]",
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}: %{GREEDYDATA:msg}",
"message", "<%{POSINT:syslog_pri}>%{UUID:esx_uuid}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}:\s%{GREEDYDATA:msg}"
]
add_tag => [ "esx" ]
}
mutate {
replace => [ "@message", "%{msg}" ]
}
if "Rejected password" in [message] and "cpu" in [message] {
grok {
break_on_match => false
match => [
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}: %{GREEDYDATA:msg}"
]
add_tag => [ "esx" ]
}
}
if "Section for VMware" in [message] {
grok {
break_on_match => false
match => [
"message", "<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp}\s%{SYSLOGHOST:esx_syslog_hostname}\s%{SYSLOGPROG:esx_syslog_program}: %{GREEDYDATA:msg},\spid=%{NUMBER:esx_pid},\sversion=%{DATA:esx_version},\sbuild=%{NUMBER:esx_build},\soption=%{WORD:esx_option}"
]
add_tag => [ "esx" ]
}
}
}
}
filter {
if [type] == "nxlog" {
json {
source => "message"
}
mutate {
lowercase => [ "EventType", "FileName", "Hostname", "Severity", "host" ]
rename => [ "Hostname", "host" ]
rename => [ "Message", "message" ]
}
}
}
The system stop to work when i add my grok filter for my fortigate firewall :
if [type] == "syslog" {
if [host] == "10.33.2.254" {
grok {
patterns_dir => "/etc/logstash/patterns/"
match => ["message","%{FORTIGATE_BASE},poluuid=%{UUID:fw_poluuid},sessionid=%{NUMBER},proto=%{WORD:fw_proto},action=%{WORD:fw_action},policyid=%{NUMBER:fw_policyid},%{FORTIGATE_COUNTRY},transip=%{IPV4:fw_transip},transport=%{NUMBER:fw_transport},service=%{QUOTEDSTRING:fw_service},%{FORTIGATE_BYTE_PKT}"]
match => ["message","%{FORTIGATE_BASE},poluuid=%{UUID:fw_poluuid},sessionid=%{NUMBER},proto=%{WORD:fw_proto},action=%{WORD:fw_action},policyid=%{NUMBER:fw_policyid},%{FORTIGATE_COUNTRY},tranip=%{IPV4:fw_transip},tranport=%{NUMBER:fw_transport},service=%{QUOTEDSTRING:fw_service},%{FORTIGATE_BYTE_PKT}"]
match => ["message","%{FORTIGATE_BASE},sessionid=%{NUMBER},proto=%{WORD:fw_proto},action=%{WORD:fw_action},policyid=%{NUMBER:fw_policyid},%{FORTIGATE_COUNTRY},service=%{QUOTEDSTRING:fw_service},app=%{QUOTEDSTRING:fw_app},%{FORTIGATE_BYTE_PKT}"]
add_tag => [ "Firewall" ]
}
}
}
i put this block behind the esx block and i the first filter block.
When i restart my logstash service, i have no more incoming data in kibana.
How can i fix this ?
Thanks