Hello All ,
We have a weird issue in ELK . We have introduced fingerprint in logstash , But still we can see the duplicates in Kibana. Kindly help how can we improve this .
input {
beats {
port => 5044
tags => ["srvr_logs"]
}
}
filter
{
if "metric" not in [fields][type]
{
mutate {
remove_field => [ "host" ]
}
if "SOAPRouter" in [source]
{
ruby {
code => "
event.set('Server', event.get('source').split('\\')[4])
"
}
mutate {
add_field => {
"Component" => "SOAPRouter"
}
}
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{SOAPROUTER}"]
}
}
else if "RequestRouter" in [source]
{
ruby {
code => "
event.set('Server', event.get('source').split('\\')[4])
event.set('InterfaceName', event.get('source').split('\\')[5])
"
}
mutate {
add_field => {
"Component" => "RequestRouter"
}
gsub => ["InterfaceName", "RequestRouter_", ""]
}
ruby {
code => "
event.set('InterfaceName', event.get('InterfaceName').split('.')[0])
"
}
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{REQUESTROUTER1}",
"message", "%{REQUESTROUTER2}"]
}
mutate
{
gsub => ["EventType", "[0-9]", ""]
gsub => ["EventType", "\[\]", ""]
remove_field => [ "NR1" ]
remove_field => [ "NR2" ]
}
ruby {
code => "
event.set('EventType', event.get('EventType').split(' ')[0])
"
}
}
else if "eappweb" not in [source] and "RequestRouter" not in [source] and "SOAPRouter" not in [source]
{
ruby {
code => "
event.set('Server', event.get('source').split('\\')[4])
event.set('Component', event.get('source').split('\\')[5])
"
}
mutate
{
gsub => ["Component", "_[0-9].*", ""]
}
if "SBL" in [message]
{
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{SERVERLOG2}"]
}
}
else
{
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{SERVERLOG1}"]
}
}
if "srvr_logs" in [tags] and "_grokparsefailure" in [tags]
{
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{SERVERLOG1}"]
}
}
}
else if "eappweb" in [source]
{
mutate {
rename => {
"[beat][hostname]" => "Server"
}
add_field => {
"Component" => "eappweb"
}
}
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{SERVERLOG1}"]
}
if "srvr_logs" in [tags] and "_grokparsefailure" in [tags]
{
grok {
patterns_dir => [ "E:\ELK\logstash\patterns\patterns.txt" ]
match => ["message", "%{SERVERLOG1}"]
}
}
}
if "RequestRouter" in [source] or "SOAPRouter" in [source]
{
date {
target => "EventDate"
match => [ "EventDate", "yyyy-MM-dd HH:mm:ss,SSS" ]
}
}
else
{
date {
target => "EventDate"
match => [ "EventDate", "yyyy-MM-dd HH:mm:ss" ]
}
}
fingerprint {
source => ["message","source"]
target => "[fingerprint]"
key => "Compass"
method => "MD5"
concatenate_sources => true
}
}
}
output {
if "PROD" in [fields][env] and "metric" not in [fields][type]{
elasticsearch {
hosts => ["https:XXXXXXXXXXX:8200"]
index => "prodsrvrlog-%{+YYYY.MM.dd}"
user => "${es_usr}"
password => "${es_pwd}"
ssl => true
ssl_certificate_verification => false
cacert => "E:\ELK\ODForESearch\config\chain.pem"
ilm_enabled => false
}
}
else if "metric" in [fields][type]{
elasticsearch {
hosts => ["https:XXXXXXXXXX8200"]
index => "metricbeat-%{+YYYY.MM.dd}"
user => "${es_usr}"
password => "${es_pwd}"
ssl => true
ssl_certificate_verification => false
cacert => "E:\ELK\ODForESearch\config\chain.pem"
ilm_enabled => false
}
}
}