Seeing 5 duplicate entries in kibana for each log entry

I have ELK cluster setup in which filebeat pulls the logs and sends to elasticsearch.
The issue is that for each entry, the kibana shows exact 5 entries.
Can someone point out the issue as to why this is happening ?

Here's my logstash configuration

input {
  beats {
    port => 5044
    add_field => { "log_level" => "-" }
    tags => [ "filebeat-log" ]
    type => "beats"
    client_inactivity_timeout => 300
  }
}
filter {
  if [type] == "beats" {
   json {
      source => "message"
    }

   if [log_type] != "nginx_logs"{
      mutate {
        rename => ["log", "log_message"]
      }
      json {
        source => "log_message"
      }
   }
  }
}
output {
  if [type] == "beats" {
    elasticsearch {
      hosts => ["http://localhost:9200"]
      index => "filebeat-%{+YYYY.MM.dd}"
      action => "index"
    }
  }
}

Blockquote

What's your cluster configuration ?

looks like it is working on adding fingerprint

filter {
  if [type] == "beats" {
   json {
      source => "message"
    }

   if [log_type] != "nginx_logs"{
      mutate {
        rename => ["log", "log_message"]
      }
      json {
        source => "log_message"
      }
   }
  }
   fingerprint {
        key => "1234ABCD"
        method => "SHA256"
        source => "message"
        target => "[@metadata][generated_id]"
        concatenate_sources => true
}
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.