Logstash reading the file line multiple times

Elastic Stack Logstash
1

Hello,

In my logstash installation, it is reading a csv file to insert it in elasticsearch. It should read each line just one time, but in fact is reading/processing up to 4 times:
{
"@version" => "1",
"endpoint" => "/managetv/tvrecord/recordings/filterByPage",
"total_response_time" => 164.0,
"send_body_ms" => 0.003814697265625,
"@timestamp" => 2019-06-11T13:40:56.521Z,
"wait_for_response_ms" => 58.86197090148926,
"dial_ms" => 33.52546691894531,
"path" => "/home/tania/runscope_csv/extract_data.csv",
"dns_lookup_ms" => 8.34965705871582,
"host" => "MAD00303",
"receive_response_ms" => 0.06461143493652344,
"code_status" => " 200",
"message" => "/managetv/tvrecord/recordings/filterByPage, mar jun 11 15:36:49 DST 2019, 200, 164.0, 70.96147537231445, 33.52546691894531, 0.003814697265625, 58.86197090148926, 8.34965705871582, 0.06461143493652344 ",
"send_headers_ms" => 70.96147537231445
}
{
"@version" => "1",
"code_status" => " 200",
"dns_lookup_ms" => 8.34965705871582,
"total_response_time" => 164.0,
"send_body_ms" => 0.003814697265625,
"endpoint" => "/managetv/tvrecord/recordings/filterByPage",
"message" => "/managetv/tvrecord/recordings/filterByPage, mar jun 11 15:36:49 DST 2019, 200, 164.0, 70.96147537231445, 33.52546691894531, 0.003814697265625, 58.86197090148926, 8.34965705871582, 0.06461143493652344 ",
"@timestamp" => 2019-06-11T13:40:56.678Z,
"send_headers_ms" => 70.96147537231445,
"path" => "/home/tania/runscope_csv/extract_data.csv",
"wait_for_response_ms" => 58.86197090148926,
"host" => "MAD00303",
"dial_ms" => 33.52546691894531,
"receive_response_ms" => 0.06461143493652344

My logshtash configuration is:

input {
file {
path => "/home/tania/runscope_csv/extract_data.csv"
codec => 'plain'
}
}

filter {
csv {
separator => ","
columns => [ "endpoint","timestamp","code_status","total_response_time","send_headers_ms","dial_ms","send_body_ms","wait_for_response_ms","dns_lookup_ms","receive_response_ms" ]
}

    mutate{
            convert => {
            "send_headers_ms" => "float"
            "send_body_ms" => "float"
            "wait_for_response_ms" => "float"
            "dns_lookup_ms" => "float"
            "receive_response_ms" => "float"
            "dial_ms" => "float"
            "total_response_time" => "float"
            }

     }
    mutate {remove_field => ["timestamp"]}

}

output {
elasticsearch {
hosts => "localhost:9200"
index => "runscope"
document_id => "%{[upc_code]}
}
stdout {}
}

Please, could you advise in which is failing there?

2

Set the number of workers in your pipeline to 1 or if you're running logstash from the command line use the -w 1 parameter

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.