Logstash slow preprocessing of data

[josephmanalo]

Hi. This is the result of my logstash statistics (curl localhost:9600/_node/stats/events)
I think it is running slow. It takes 1 doc/28 sec to process it.

Here are some details:
~ Aprrox. 30M of data a day
~ Machine specs is i3, 24GB RAM, 2TB
~ There I set logstash workers into 12

Could you help me with my problem?

Here's the result:

root@ThinkPad-L470:/etc/logstash# curl -XGET localhost:9600/_node/stats/events?pretty=true
{
  "host" : "ThinkPad-L470",
  "version" : "6.4.2",
  "http_address" : "127.0.0.1:9600",
  "id" : "6b434b8f-9ec4-4a7b-863f-0c7f707a72e9",
  "name" : "-ThinkPad-L470",
  "events" : {
    "in" : 705374,
    "filtered" : 703870,
    "out" : 702370,
    "duration_in_millis" : 25165661,
    "queue_push_duration_in_millis" : 8211080
  }
}root@-ThinkPad-L470:/etc/logstash# curl -XGET localhost:9600/_node/stats/events?pretty=true
{
  "host" : "-ThinkPad-L470",
  "version" : "6.4.2",
  "http_address" : "127.0.0.1:9600",
  "id" : "6b434b8f-9ec4-4a7b-863f-0c7f707a72e9",
  "name" : "-ThinkPad-L470",
  "events" : {
    "in" : 803999,
    "filtered" : 802495,
    "out" : 800995,
    "duration_in_millis" : 28264950,
    "queue_push_duration_in_millis" : 9221083
  }
}root@-ThinkPad-L470:/etc/logstash# curl -XGET localhost:9600/_node/stats/events?pretty=true
{
  "host" : "-ThinkPad-L470",
  "version" : "6.4.2",
  "http_address" : "127.0.0.1:9600",
  "id" : "6b434b8f-9ec4-4a7b-863f-0c7f707a72e9",
  "name" : "-ThinkPad-L470",
  "events" : {
    "in" : 942249,
    "filtered" : 940745,
    "out" : 939245,
    "duration_in_millis" : 33247715,
    "queue_push_duration_in_millis" : 10844023
  }
}root@-ThinkPad-L470:/etc/logstash# curl -XGET localhost:9600/_node/stats/events?pretty=true
{
  "host" : "-ThinkPad-L470",
  "version" : "6.4.2",
  "http_address" : "127.0.0.1:9600",
  "id" : "6b434b8f-9ec4-4a7b-863f-0c7f707a72e9",
  "name" : "-ThinkPad-L470",
  "events" : {
    "in" : 1008249,
    "filtered" : 1006620,
    "out" : 1005245,
    "duration_in_millis" : 35434236,
    "queue_push_duration_in_millis" : 11552825

Here's my beats configuration:

Here's the beats configuration
beats.conf

input{
 beats{
  port => 5044
 }
}

filter{
 if [module_name] == "match1"{
 csv {
  separator => ","

  columns => ["col1", "col2", "col3", "col4",
  "col5" ]
 }
 }

 if [module_name] == "match2"{
 csv {
  separator => ","

  columns => ["col1", "col2", "col3", "col4",
  "col5" ]
 }
 }

if [module_name] == "match3"{
 csv {
  separator => ","

  columns => ["col1", "col2", "col3", "col4",
  "col5" ]
 }
 }

output {

 if [@metadata][beat] == "winlogbeat"
        {
                 elasticsearch {
                        hosts => ["ip:address"]
                        index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
                }
        }

 if [module_name] == "match1" or [module_name] == "match2"  or [module_name] == "match3"

        {
        elasticsearch {
                hosts => ["ip:port"]
                index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
        }
 }

  if [module_name] == "match4" or [module_name] == "match5"  or [module_name] == "match6"
        {
         elasticsearch {
                hosts => ["ip:port"]
                index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
                }
        }

 if [module_name] == "match7" or [module_name] == "match8"  or [module_name] == "match9"{
                elasticsearch {
                        hosts => ["ip:port"]
                        index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"}
        }