Hello Team,
I am using ELK6.4.0 and filebeat6.4.0. My architecture is Filebeat->Logastash->Elasticsearch->Kibana.
Currently i have 2 index metrcibeat-* and filebeat-* on kibana. In metricbeat index i am getting our application logs as well as nginx, syslog and auth log.
But now we want to create separate index for all, so we can search our logs easily.
I have tried the below config in logstash but no success. Below is my logstash config:-
input {
beats {
port => 5044
ssl => true
ssl_certificate_authorities => ["/etc/pki/tls/ca.crt"]
ssl_certificate => "/etc/pki/tls/server.crt"
ssl_key => "/etc/pki/tls/server.key"
ssl_verify_mode => "peer"
tls_min_version => "1.2"
}
}
filter {
grok {
match => { "message" => [ "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}", "\I\,\s\[(?<date-time>[\d\-\w\:\.]+)\s\#(?<pid>\d+)\]\s+(?<loglevel>\w+)\s\-+\s\:\s\[(?<request-id>[\d\w\-]+)\]\s(?<method>[\w\s]+)\s\"(?<path>[\w\/\.]+)\"\s(?<mlp-message>.*)", "\I\,\s\[(?<date-time>[\d\-\w\:\.]+)\s\#(?<pid>[\d]+)\]\s\s(?<loglevel>[\w]+)\s\--\s\:\s\[(?<request-id>[\d\-\w]+)\]\s(?:[cC]urrent\s)?[dD]evice[\s:]+(?<device-id>[\w\s\:]+)", "\I\,\s\[(?<date-time>[\d\-\w\:\.]+)\s\#(?<pid>\d+)\]\s+(?<loglevel>\w+)\s\-+\s\:\s\[(?<request-id>[\d\w\-]+)\]\s(?<mlp-message>.*)", "\w\,\s\[(?<date-time>[\w\-\:\.]+)\s\#(?<pid>\d+)\]\s+(?<loglevel>\w+)\s(?<mlp-message>.*)" ] }
add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_from", "%{host}" ]
}
}
output {
if [source] in ["/var/log/nginx/access.log", "/var/log/nginx/error.log"]{
elasticsearch {
hosts => ["10.133.58.12:9200"]
sniffing => true
# manage_template => false
index => "nginx-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
else if [source] =~ "/var/log/syslog"{
elasticsearch {
hosts => ["10.133.58.12:9200"]
sniffing => true
# manage_template => false
index => "syslog-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
else if [source] =~ "/var/log/auth.log"{
elasticsearch {
hosts => ["10.133.58.12:9200"]
sniffing => true
# manage_template => false
index => "access-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
else {
elasticsearch {
hosts => ["10.133.58.12:9200"]
sniffing => true
manage_template => false
# index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
index => "application-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
}
I am using filebeat system and nginx modules.
Please help me to troubleshoot the issue.
Thanks in advance.