for payment URL index hits are coming after every one min for 2 different urls.
i.e for each url we are getting 60 hits per min.
we have created sincedb path for index and observed data but we found that data is duplicating sometimes without taking restart of kibana services
As attached in below screenshot we can see sudden rise in the graph.
Please find configuration file for index as below.
input {
file
{
path => ["/log/AnsibleLogs/Formatfile.txt"]
tags => ["paymenturls"]
# sincedb_path => "/dev/null"
# start_position => "end"
# ignore_older => "3m"
sincedb_path => "/var/lib/logstash/plugins/inputs/file/.sincedb_9e5a6659de603250b13193c70cd50ef7"
}
}
filter{
if "paymenturls" in [tags]{
grok {
match => { "message" => "%{DATA:url}, %{DAY:day} %{MONTH:month} (?[0-9 ]{2}) %{TIME:time} %{WORD:Zone} %{YEAR:year}, %{NUMBER:Status_code1}, %{GREEDYDATA:ErrorMessage}" }
}
mutate {
add_field => {
"timestamp" => "%{date} %{month} %{year} %{time}"
}
}
}
}
output
{
if "paymenturls" in [tags]
{
elasticsearch
{
hosts => [ "https://elkmonp4.newindia.co.in:9200"]
index => "niapaymenturls-%{+YYYY.MM.dd}"
ssl => true
cacert => ["/etc/logstash/ELKMONP4.crt"]
user => "elastic"
password => "Password@123"
}
}
}