Hi experts.
I already setup ELK stack and it working normaly. with my grok filter
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:Time} %{WORD:Method} %{NUMBER:Http_stt} %{NUMBER:ResTime}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
i Recieve all the field in kibana. Now i have more log to control so i added tags in filebeat config
- input_type: log
paths:
- /root/DemoVCC/test.log
fields:
tags: node
- input_type: log
paths:
- /var/log/mysqld.log*
exclude_files: [".gz$"]
fields:
tags: mysql-error
and my new grok filter:
filter {
if "node" in [tags] {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601: Time} %{WORD: Method} %{NUMBER: Http_stt} %{NUMBER:ResTime}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
if "mysql-error" in [tags] {
grok {
match => { "message" => ["%{LOCALDATETIME:[mysql][error][timestamp]} (\[%{DATA:[mysql][error][level]}\] )?%{GREEDYDATA:[mysql][error][message]}",
"%{TIMESTAMP_ISO8601:[mysql][error][timestamp]} %{NUMBER:[mysql][error][thread_id]} \[%{DATA:[mysql][error][level]}\] %{GREEDYDATA:[mysql][error][message1]}",
"%{GREEDYDATA:[mysql][error][message2]}"] }
pattern_definitions => {
"LOCALDATETIME" => "[0-9]+ %{TIME}"
}
remove_field => "message"
}
mutate {
rename => { "[mysql][error][message1]" => "[mysql][error][message]" }
}
mutate {
rename => { "[mysql][error][message2]" => "[mysql][error][message]" }
}
date {
match => [ "[mysql][error][timestamp]", "ISO8601", "YYMMdd H:m:s" ]
remove_field => "[apache2][access][time]"
}
}
}
after that i still recieve the filebeat log but nolonger see the Time, Method, Http_stt, ResTime in kibana field.
Can anyone help me with this.
Thanks