I am getting 3 log files from the FileBeat on 4 nodes. Only 1 log from single node is getting indexed in elasticsearch.
input {
beats {
port => 5044
ssl => true
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
filter {
if [fields][log_type] == "namenode" {
grok {
match => {
"message" => "%{DATESTAMP:log_timestamp} %{LOGLEVEL:log_level} (?<log_action>\b\w+(\.\w+)?\b) (%{DATA:log_actor}) - %{GREEDYDATA:log_message}"
}
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
target => "log_timestamp"
}
}
else if [fields][log_type] == "namenode_audit" {
grok {
match => {
"message" => "%{DATESTAMP:log_timestamp} %{LOGLEVEL:log_level} (?:\b\w+(\.\w+)?\b): allowed=(?<allowed>true|false) ugi=%{DATA:user} (%{DATA:authentication_type}) ip=/%{IPV4:host_requested} cmd=%{DATA:command} src=%{DATA:source} dst=%{DATA:destination} perm=%{DATA:permission} proto=%{GREEDYDATA:protocol}"
}
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
target => "log_timestamp"
}
}
else if [fields][log_type] == "resource_manager" {
grok {
match => {
"message" => "%{GREEDYDATA:data}"
}
add_field => [ "received_from", "%{host}" ]
}
date {
match => [ "log_timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
target => "log_timestamp"
}
}
}
output {
elasticsearch {
hosts => "localhost:9200"
index => "%{[fields][log_type]}-%{[beat][hostname]}"
manage_template => false
}
}