Getting Data duplicated while parsing the logs using Logstash

[Rajat_Singh]

Hello Folks,

I am using Logstash for the purpose of parsing the logs to elasticsearch as their were 4 files each of 4 gb ,It took around one day it to parse all the value.
so my first concern is that what things in can do to improve the parsing speed
second concern is that i don't know how the data from one of the file got duplicated.

[jsvd (João Duarte)]

can you show the configuration you used?

[Rajat_Singh]

input
{
file {
path => ["/root/logstash/logstash-2.4.0/bin/Logs/one/disp1.log"]
start_position => "beginning"
type => "Disp1"
}

file
{
path => ["/root/logstash/logstash-2.4.0/bin/Logs/two/disp2.log"]
start_position => "beginning"
type => "Disp2"
}

file
{
path => ["/root/logstash/logstash-2.4.0/bin/Logs/three/disp3.log"]
start_position => "beginning"
type => "Disp3"
}

file
{
path => ["/root/logstash/logstash-2.4.0/bin/Logs/four/disp4.log"]
start_position => "beginning"
type => "Disp4"
}

}
filter
{
if [type] == "Disp1" {
if "health" in [message] { drop{} }

  else
  {

grok {
match => { message => [
"%{IPV4:ip} - - (?[%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year}:%{TIME:time} -%{INT:seconds}]) "%{WORD:method} %{NOTSPACE:url} %{NOTSPACE:protocol}" %{INT:http_code} %{INT:download_byte} "%{DATA:refer}" "%{DATA:user_agent}" "%{NOTSPACE:transaction_name}" %{INT:response_time_sec} %{INT:response_time}"

                  ]}
				  
				  
		add_tag => [ "Disp1" ]

}

mutate {
convert => { "http_code" => "integer"}
convert => { "download_byte" => "integer"}
convert => { "response_time_sec" => "integer"}
convert => { "response_time" => "integer"}
}

							ruby {

code => "event.set('response_time_sec', event.get('response_time_sec').to_f / 1000000)"
}

			}

}

if [type] == "Disp2" {
if "health" in [message] { drop{} }

  else
  {

grok {
match => { message => [
"%{IPV4:ip} - - (?[%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year}:%{TIME:time} -%{INT:seconds}]) "%{WORD:method} %{NOTSPACE:url} %{NOTSPACE:protocol}" %{INT:http_code} %{INT:download_byte} "%{DATA:refer}" "%{DATA:user_agent}" "%{NOTSPACE:transaction_name}" %{INT:response_time_sec} %{INT:response_time}"

                  ]}
				  
				  
		add_tag => [ "Disp2" ]

}

mutate {
convert => { "http_code" => "integer"}
convert => { "download_byte" => "integer"}
convert => { "response_time_sec" => "integer"}
convert => { "response_time" => "integer"}
}
ruby {
code => "event.set('response_time_sec', event.get('response_time_sec').to_f / 1000000)"
}

			}

}

if [type] == "Disp3" {
if "health" in [message] { drop{} }

  else
  {

grok {
match => { message => [
"%{IPV4:ip} - - (?[%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year}:%{TIME:time} -%{INT:seconds}]) "%{WORD:method} %{NOTSPACE:url} %{NOTSPACE:protocol}" %{INT:http_code} %{INT:download_byte} "%{DATA:refer}" "%{DATA:user_agent}" "%{NOTSPACE:transaction_name}" %{INT:response_time_sec} %{INT:response_time}"

                  ]}
				  
				  
		add_tag => [ "Disp3" ]

}

mutate {
convert => { "http_code" => "integer"}
convert => { "download_byte" => "integer"}
convert => { "response_time_sec" => "integer"}
convert => { "response_time" => "integer"}
}
ruby {
code => "event.set('response_time_sec', event.get('response_time_sec').to_f / 1000000)"
}

			}

}

if [type] == "Disp4" {
if "health" in [message] { drop{} }

  else
  {

grok {
match => { message => [
"%{IPV4:ip} - - (?[%{MONTHDAY:day}/%{MONTH:month}/%{YEAR:year}:%{TIME:time} -%{INT:seconds}]) "%{WORD:method} %{NOTSPACE:url} %{NOTSPACE:protocol}" %{INT:http_code} %{INT:download_byte} "%{DATA:refer}" "%{DATA:user_agent}" "%{NOTSPACE:transaction_name}" %{INT:response_time_sec} %{INT:response_time}"

                  ]}
				  
				  add_tag => [ "Disp4" ]

}

mutate {
convert => { "http_code" => "integer"}
convert => { "download_byte" => "integer"}
convert => { "response_time_sec" => "integer"}
convert => { "response_time" => "integer"}
}
ruby {
code => "event.set('response_time_sec', event.get('response_time_sec').to_f / 1000000)"
}
}
}

}

output
{
stdout {codec => rubydebug}

if "Disp1" in [tags] {
elasticsearch { hosts => ["100.150.86.175:9200"] index => "logstash-altria-test"}}

if "Disp2" in [tags] {
elasticsearch { hosts => ["100.150.86.175:9200"] index => "logstash-altria-test"}}

if "Disp3" in [tags] {
elasticsearch { hosts => ["100.150.86.175:9200"] index => "logstash-altria-test"}}

if "Disp4" in [tags]{
elasticsearch { hosts => ["100.150.86.175:9200"] index => "logstash-altria-test"}}
}