Optimize grok filter

JeanM · August 12, 2019, 7:20pm

Hello,

I search some help in order to optimize grok filter.

Actually, i have 100% CPU used.

Have you some tricks ?

Sincerely

Badger · August 12, 2019, 7:28pm

Show us the configuration of your grok filter and an example of the data it is trying to match.

High CPU is typically caused by excessive backtracking. Anchoring expressions can help, as can replacing DATA and especially GREEDYDATA with more specific regexps.

JeanM · August 13, 2019, 6:22am

Filter configuration :

filter {

  if "inwebo_access" in [tags] or "nginx-access" in [tags] {

    grok {

      match => { "message" => "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG} %{IPORHOST:nginx_source}(?:%{NOTSPACE})? - %{NOTSPACE:nginx_auth} \[%{HTTPDATE:nginx_timestamp}\](%{SPACE}*)\"%{WORD:nginx_method} %{URIPATHPARAM:nginx_uri} HTTP/%{NUMBER:nginx_httpversion}(( Host:)?%{IPORHOST:nginx_host_header})?\" %{NUMBER:nginx_status} %{NUMBER:nginx_bytes} %{QS:nginx_referrer} %{QS:nginx_agent} IP_FORWARDEE \"(%{IP:nginx_ip_forwardee}|-)\" IP_PROXY_FORWARDEE \"%{IP:nginx_ip_proxy_forwardee}(, %{IP:nginx_ip_proxy_forwardee_list})?\"( \"(%{IP:nginx_ip_proxy_forwardee_list}(:%{NUMBER})?|-)\")?$"}
    }
  }

  if [nginx_ip_proxy_forwardee] {
      if [nginx_ip_proxy_forwardee] !~ /localhost|\-/ {
        cidr {
          add_tag => [ "internalIP" ]
          address => [ "%{nginx_ip_proxy_forwardee}" ]
          network => [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16","100.64.0.0/10","127.0.0.1" ]
        }
      }

      if "internalIP" not in [tags] {
        geoip {
          source => "nginx_ip_proxy_forwardee"
          database => "/etc/logstash/geoip/GeoLite2-City.mmdb"
        }
      }
    }

  if "nginx" in [tags] {

    grok {

      match => { "message" => "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG} (?<nginx_timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:nginx_severity}\] %{POSINT:nginx_pid}#%{NUMBER}: %{GREEDYDATA:nginx_message}$" }

    }

  }


  if "logs_system" in [tags] {

    grok {

      match => { "message" => "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG}: (\[%{LOGLEVEL:syslog_level}\])?%{GREEDYDATA:syslog_message}$" }

    }

  }

  if "mailbox" in [tags] {

    grok {

      patterns_dir => ["/etc/logstash/patterns"]
      patterns_files_glob => "zimbra"
      match => { "message" => "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG} (%{TIMESTAMP_ISO8601:mailbox_timestamp} %{LOGLEVEL:mailbox_loglevel}(%{SPACE})*\[%{DATA:mailbox_thread}\] \[name=%{EMAILADDRESS:email}(;aname=%{EMAILADDRESS:email_aname})?(;mid=%{NUMBER:mid})?(;oip=%{IPORHOST:mailbox_origin_host})?(;ip=%{IPORHOST:mailbox_source})?(;port=%{NUMBER:port})?(;ua=%{USERAGENT:mailbox_ua})?(;via=%{IPORHOST:audit_via}\(%{USERAGENT:audit_via_ua}\)(,%{IPORHOST:audit_via}\(%{USERAGENT:audit_via_ua}\))*)?(;soapId=%{USER:mailbox_soapid})?;\] %{WORD:mailbox_protocole} - %{GREEDYDATA:mailbox_message}|%{GREEDYDATA:mailbox_trace})$" }

    }

  }


  if "audit" in [tags] {

    grok {

      patterns_dir => ["/etc/logstash/patterns"]
      match => { "message" => "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG} %{TIMESTAMP_ISO8601:audit_timestamp} %{LOGLEVEL:audit_loglevel}(%{SPACE})*\[%{DATA:audit_thread}\] \[(name=%{EMAILADDRESS:email};)?(aname=%{EMAILADDRESS:email_aname};)?(mid=%{NUMBER:mid};)?(ip=%{IPORHOST:audit_source}(, %{IPORHOST:audit_source})*;)?(oip=%{IPORHOST:audit_origin_host}(, %{IPORHOST:audit_origin_host_list})*;)?(port=%{NUMBER:port};)?(via=%{IPORHOST:audit_via}\(%{AUDIT_USERAGENT:audit_via_ua}\);)?(ua=%{AUDIT_USERAGENT:audit_ua};)?(soapId=%{USER:audit_soapid};)?\] %{WORD:audit_category} - (cmd=%{WORD:audit_action})?(%{AUDIT_DETAILS}|%{GREEDYDATA:audit_divers})$" }

    }

  if [audit_origin_host] {
      if [audit_origin_host] !~ /localhost|\-/ {
        cidr {
          add_tag => [ "internalIP" ]
          address => [ "%{audit_origin_host}" ]
          network => [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16","100.64.0.0/10","127.0.0.1" ]
        }
      }

      if "internalIP" not in [tags] {
        geoip {
          source => "audit_origin_host"
          database => "/etc/logstash/geoip/GeoLite2-City.mmdb"
        }
      }
    }

  }


  if "zimbra" in [tags] {

    grok {

      patterns_dir => ["/etc/logstash/patterns"]
      patterns_files_glob => "zimbra"
      match => { "message" => [ "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG}: %{AMAVIS}$", "^%{MAILLOG}$", "^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG}: %{GREEDYDATA:zimbra_message}$"] }
    }

    if [amavis_sender] =~ '/.+/' {
        grok {
          match => { "amavis_sender" => "%{USERNAME:sender_username}@%{HOSTNAME:sender_domain}" }
          match => { "amavis_recipient" => "%{USERNAME:recipient_username}@%{HOSTNAME:recipient_domain}"  }

        }
    }

  }



  mutate {
    remove_field => [ "agent", "host", "log", "ecs", "input" ]
  }


}

JeanM · August 13, 2019, 6:22am

Patterns :

EMAILADDRESS %{USER:email_username}@%{HOSTNAME:email_domain}
AUDIT_ACCOUNT (account|name)=%{EMAILADDRESS:audit_account}
AUDIT_PROTOCOL protocol=%{WORD:audit_protocol}
AUDIT_ERROR error=%{GREEDYDATA:audit_error}
AUDIT_DETAILS (; %{AUDIT_ACCOUNT})?(; %{AUDIT_PROTOCOL})?(; %{AUDIT_ERROR})?(; %{GREEDYDATA:audit_divers})?
AUDIT_USERAGENT [a-zA-Z0-9( );:/._-]+


#####################################
#
# amavis
AMAVIS_THREAD  ([0-9]*-[0-9]*)
AMAVIS_QUEUEID    (?:[0-9A-F]{6,}|[0-9a-zA-Z]{15,})
AMAVIS_STATUS     (?:Passed|Blocked)
AMAVIS_ORIGINATING (?:ORIGINATING|ORIGINATING_POST)(?:/MYNETS)?
#
EMAIL %{USERNAME:email_username}@%{HOSTNAME:email_domain}
#
AMAVIS_INIT (?:starting\.|perl=|SpamControl:|Net::Server:|Module|SQL::Quarantine|%{GREEDYDATA} code |Found|No|Internal|Using|initializing|extra)
AMAVIS_STARTUP ^%{AMAVIS_INIT}
AMAVIS_REEXEC ^\(\!\)Net::Server: %{GREEDYDATA} Re-exec server during HUP
AMAVIS_EXTRA \(%{AMAVIS_THREAD:amavis_thread}\) extra modules loaded: %{GREEDYDATA:amavis_extra_module}
AMAVIS_PREPARE \(%{AMAVIS_THREAD:amavis_thread}\) TempDir::prepare_file: %{GREEDYDATA:amavis_prepare}
AMAVIS_UNZIP \(%{AMAVIS_THREAD:amavis_thread}\) (?:\(\!\)|)do_unzip: %{GREEDYDATA:amavis_unzip_error}
AMAVIS_ASCII \(%{AMAVIS_THREAD:amavis_thread}\) do_ascii: %{GREEDYDATA:amavis_ascii_error}
AMAVIS_INFO \(%{AMAVIS_THREAD:amavis_thread}\) INFO: %{GREEDYDATA:amavis_info}
AMAVIS_WARN \(%{AMAVIS_THREAD:amavis_thread}\) WARN: %{GREEDYDATA:amavis_warning}
AMAVIS_LOCAL_DELIVERY \(%{AMAVIS_THREAD:amavis_thread}\) local delivery: %{GREEDYDATA:amavis_local_delivery}
AMAVIS_SA \(%{AMAVIS_THREAD:amavis_thread}\) SA info: %{GREEDYDATA:amavis_sa_info}
AMAVIS_FILECHECK \(%{AMAVIS_THREAD:amavis_thread}\) ESMTP (\[%{IP:amavis_source_ip}\])?:%{POSINT:amavis_relay_port} %{DATA:amavis_quarantine_file}:\s+<(?:%{EMAIL:amavis_sender})?> -> <(?:%{EMAIL:amavis_recipient})?>(?:,<%{EMAIL:amavis_recipient}>)*(?: SIZE=%{POSINT:amavis_size})?(?: RET=%{DATA:amavis_ret})?(?: BODY=%{DATA:amavis_body})?(?: ENVID=%{DATA:amavis_envid})? Received: %{GREEDYDATA:amavis_summary}
AMAVIS_CHECKING \(%{AMAVIS_THREAD:amavis_thread}\) Checking: %{DATA:amavis_mail_id} (?:%{AMAVIS_ORIGINATING} )?(?:\[%{IP:amavis_source_ip}\] )?<(?:%{EMAIL:amavis_sender})?> -> <(?:%{EMAIL:amavis_recipient})?>(?:,<%{EMAIL:amavis_recipient}>)*
AMAVIS_CHECKRELAY \(%{AMAVIS_THREAD:amavis_thread}\) Open relay\? Nonlocal recips but not originating: %{EMAIL:amavis_recipient}
AMAVIS_QUEUED \(%{AMAVIS_THREAD:amavis_thread}\) %{DATA:amavis_mail_id} (FWD|SEND) from <(?:%{EMAIL:amavis_sender})?> -> <(?:%{EMAIL:amavis_recipient})?>,( )?(?:ENVID=%{DATA:amavis_envid})?(?:RET=%{DATA:amavis_ret})?(?:BODY=%{DATA:amavis_body})? 250 2.0.0 from MTA\(smtp:\[%{IP:amavis_source_ip}\]:%{POSINT:amavis_relay_port}\): 250 2.0.0 Ok: queued as %{AMAVIS_QUEUEID:amavis_queued_as}
AMAVIS_RESULT \(%{AMAVIS_THREAD:amavis_thread}\) %{AMAVIS_STATUS:amavis_status} %{DATA:amavis_result} (?:\(%{DATA:amavis_result_summary}\) )?\{%{DATA:amavis_actions}\}, (?:(?:(?:(?:%{AMAVIS_ORIGINATING} LOCAL) )?\[%{IP:amavis_relay_ip}\](?::%{POSINT:amavis_relay_port})?( \[%{IP:amavis_origin_ip}\])?)? )?<(?:%{EMAIL:amavis_sender})?> -> <(?:%{EMAIL:amavis_recipient})?>(?:,<%{EMAIL:amavis_recipient}>)*, (?:quarantine: %{DATA:amavis_quarantine}, )?(?:Queue-ID: %{AMAVIS_QUEUEID:amavis_queueid}, )?Message-ID: <(?:%{DATA:amavis_message-id})?>, (?:mail_id: %{DATA:amavis_mail_id}, )?(?:Hits: %{DATA:amavis_hits}, )?(?:size: %{NONNEGINT:amavis_size}, )?(?:queued_as: %{AMAVIS_QUEUEID:amavis_queued_as}(?:/%{AMAVIS_QUEUEID:amavis_queued_as2})?, )?(?:dkim_sd=%{DATA:amavis_dkim_sd}, )?%{NONNEGINT:amavis_delay} ms
#
# ###
AMAVIS (%{AMAVIS_STARTUP}|%{AMAVIS_REEXEC}|%{AMAVIS_EXTRA}|%{AMAVIS_PREPARE}|%{AMAVIS_UNZIP}|%{AMAVIS_ASCII}|%{AMAVIS_INFO}|%{AMAVIS_WARN}|%{AMAVIS_LOCAL_DELIVERY}|%{AMAVIS_SA}|%{AMAVIS_FILECHECK}|%{AMAVIS_CHECKING}|%{AMAVIS_CHECKRELAY}|%{AMAVIS_QUEUED}|%{AMAVIS_RESULT})

JeanM · August 13, 2019, 6:23am

COMPONENT ([\w._\/%-]+)
COMPID postfix\/%{COMPONENT:component}(?:\[%{NUMBER:pid}\])?
POSTFIX (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{COMPID}:

HELO (?:\[%{IP:helo}\]|%{HOSTNAME:helo}|%{DATA:helo})

MILTERCONNECT %{QUEUEID:qid}: milter-reject: CONNECT from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; proto=%{WORD:proto}
MILTERUNKNOWN %{QUEUEID:qid}: milter-reject: UNKNOWN from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; proto=%{WORD:proto}
MILTEREHLO %{QUEUEID:qid}: milter-reject: EHLO from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; proto=%{WORD:proto} helo=<%{HELO}>
MILTERMAIL %{QUEUEID:qid}: milter-reject: MAIL from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; from=<%{EMAIL:from}> proto=%{WORD:proto} helo=<%{HELO}>
MILTERHELO %{QUEUEID:qid}: milter-reject: HELO from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; proto=%{WORD:proto} helo=<%{HELO}>
MILTERRCPT %{QUEUEID:qid}: milter-reject: RCPT from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; from=<%{EMAIL:from}> to=<%{EMAIL:to}> proto=%{WORD:proto} helo=<%{HELO}>
MILTERENDOFMESSAGE %{QUEUEID:qid}: milter-reject: END-OF-MESSAGE from %{RELAY:relay}: %{GREEDYDATA:milter_reason}; from=<%{EMAIL:from}> to=<%{EMAIL:to}> proto=%{WORD:proto} helo=<%{HELO}>

QUEUEID (?:[A-F0-9]+|NOQUEUE)
RELAY (?:%{HOSTNAME:relayhost}(?:\[%{IP:relayip}\](?::[0-9]+(.[0-9]+)?)?)?)
POSREAL [0-9]+(.[0-9]+)?
DSN %{NONNEGINT}.%{NONNEGINT}.%{NONNEGINT}
STATUS sent|deferred|bounced|expired
PERMERROR 5[0-9]{2}
MESSAGELEVEL reject|warning|error|fatal|panic

POSTFIXSMTPMESSAGE %{MESSAGELEVEL}: %{GREEDYDATA:reason}
POSTFIXACTION discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn

POSTFIXSMTP %{POSTFIXSMTPRELAY}|%{POSTFIXSMTPCONNECT}|%{POSTFIXSMTP5XX}|%{POSTFIXSMTPREFUSAL}|%{POSTFIXSMTPLOSTCONNECTION}|%{POSTFIXSMTPTIMEOUT}
POSTFIXSMTPRELAY %{QUEUEID:qid}: to=<%{DATA:to}>,(?:\sorig_to=<%{DATA:orig_to}>,)? relay=%{RELAY},(?: delay=%{POSREAL:delay},)?(?: delays=%{DATA:delays}?,)?(?: conn_use=%{POSREAL:conn_use},)?( %{WORD}=%{DATA},)+? dsn=%{DSN:dsn}, status=%{STATUS:result} %{GREEDYDATA:reason}
POSTFIXSMTPCONNECT connect to %{RELAY}: %{GREEDYDATA:reason}
POSTFIXSMTP5XX %{QUEUEID:qid}: to=<%{EMAIL:to}>,(?:\sorig_to=<%{EMAIL:orig_to}>,)? relay=%{RELAY}, (%{WORD}=%{DATA},)+ dsn=%{DSN:dsn}, status=%{STATUS:result} \(host %{HOSTNAME}\[%{IP}\] said: %{PERMERROR:responsecode} %{DATA:smtp_response} \(in reply to %{DATA:command} command\)\)
POSTFIXSMTPREFUSAL %{QUEUEID:qid}: host %{RELAY} refused to talk to me: %{GREEDYDATA:reason}
POSTFIXSMTPLOSTCONNECTION %{QUEUEID:qid}: lost connection with %{RELAY} while %{GREEDYDATA:reason}
POSTFIXSMTPTIMEOUT %{QUEUEID:qid}: conversation with %{RELAY} timed out while %{GREEDYDATA:reason}

POSTFIXSMTPD %{POSTFIXSMTPDCONNECTS}|%{POSTFIXSMTPDMILTER}|%{POSTFIXSMTPDACTIONS}|%{POSTFIXSMTPDTIMEOUTS}|%{POSTFIXSMTPDLOGIN}|%{POSTFIXSMTPDCLIENT}|%{POSTFIXSMTPDNOQUEUE}|%{POSTFIXSMTPDWARNING}|%{POSTFIXSMTPDLOSTCONNECTION}
POSTFIXSMTPDCONNECTS (?:dis)?connect from %{RELAY}
POSTFIXSMTPDMILTER %{MILTERCONNECT}|%{MILTERUNKNOWN}|%{MILTEREHLO}|%{MILTERMAIL}|%{MILTERHELO}|%{MILTERRCPT}
POSTFIXSMTPDACTIONS %{QUEUEID:qid}: %{POSTFIXACTION:postfix_action}: %{DATA:command} from %{RELAY}: %{PERMERROR:responsecode} %{DSN:dsn} %{DATA}: %{DATA:reason}; from=<%{EMAIL:from}> to=<%{EMAIL:to}> proto=%{DATA:proto} helo=<%{HELO}>
POSTFIXSMTPDTIMEOUTS timeout after %{DATA:command} from %{RELAY}
POSTFIXSMTPDLOGIN %{QUEUEID:qid}: client=%{DATA:client}, sasl_method=%{DATA:saslmethod}, sasl_username=%{GREEDYDATA:saslusername}
POSTFIXSMTPDCLIENT %{QUEUEID:qid}: client=%{GREEDYDATA:client}
POSTFIXSMTPDNOQUEUE NOQUEUE: %{POSTFIXACTION:postfix_action}: %{DATA:command} from %{RELAY}: %{GREEDYDATA:reason}
POSTFIXSMTPDWARNING warning:( %{IP}: | hostname %{HOSTNAME} )?%{GREEDYDATA:reason}
POSTFIXSMTPDLOSTCONNECTION (?:lost connection after %{DATA:smtp_response} from %{RELAY}|improper command pipelining after HELO from %{GREEDYDATA:reason})

POSTFIXCLEANUP %{POSTFIXCLEANUPMESSAGE}|%{POSTFIXCLEANUPMILTER}
POSTFIXCLEANUPMESSAGE %{QUEUEID:qid}: (resent-)?message-id=(<)?%{GREEDYDATA:messageid}(>)?
POSTFIXCLEANUPMILTER %{MILTERENDOFMESSAGE}

POSTFIXBOUNCE %{QUEUEID:qid}: sender (non-)?delivery( status)? notification: %{QUEUEID:bouncequeueid}

POSTFIXQMGR %{QUEUEID:qid}: (?:removed|from=<(?:%{DATA:from})?>(?:, size=%{NUMBER:size}, nrcpt=%{NUMBER:nrcpt} \(%{GREEDYDATA:queuestatus}\))?)

POSTFIXANVIL statistics: %{GREEDYDATA:reason}

POSTFIXREWRITE warning: do not list domain %{DATA:domain} in BOTH mydestination and virtual_alias_domains

USER_AGENT User-Agent|X-Mailer
RECIPIENTS <%{EMAIL:recipient}>(,<%{GREEDYDATA:recipientlist}>)?
ORIGIN (%{DATA:originating_net} )\[%{IP:relay}\](:%{NUMBER}) \[%{IP:originip}\]

DOVEIMAP imap\(%{DATA:user}\): %{DATA:reason} in=%{NUMBER:inbytes} out=%{NUMBER:outbytes}

DOVECMD anvil|auth|config|log|master
DOVEMISC %{DOVECMD:command}: %{GREEDYDATA:reason}

DOVELOGIN imap-login: %{DATA:action}:(?: user=<(%{DATA:user})?>, (method=%{DATA:loginmethod}, )?rip=%{IP:rip}, lip=%{IP:lip},( mpid=%{NUMBER:mpid},( %{DATA:sectype},)?| %{DATA:securesession},)? session=<%{DATA:session}>| %{GREEDYDATA:reason})

DOVELDA lda\((%{DATA:user})?\):( %{DATA:action}:)? msgid=(?:<%{DATA:mesgid}@%{DATA:domain}>|%{DATA:mesgid}):( saved mail to| stored mail into mailbox) .*?%{DATA:folder}.*?

DOVEAUTH auth-worker\(%{NUMBER:pid}\): pam\((?:%{USERNAME:user}|%{EMAIL:user}),%{IP:ip}\): %{GREEDYDATA:reason}

DOVECOT (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} dovecot: (%{DOVEIMAP}|%{DOVELOGIN}|%{DOVELDA}|%{DOVEAUTH}|%{DOVEMISC})

PF %{POSTFIX} (?:%{POSTFIXSMTP}|%{POSTFIXANVIL}|%{POSTFIXQMGR}|%{POSTFIXBOUNCE}|%{POSTFIXCLEANUP}|%{POSTFIXSMTPD}|%{POSTFIXREWRITE})

MAILLOG (%{PF}|%{DOVECOT})

USERAGENT [a-zA-Z0-9() /._-]+

JeanM · August 13, 2019, 6:53am

Example of logs : https://justpaste.it/logstash_example

Badger · August 13, 2019, 2:09pm

You have too much alternation. Take some of that alternation out of the patterns and put it into the filter configuration.

The access logs should be fine. The mailbox, audit, and other similar logs will be where the problem is.

Looking at the mailbox pattern, you have a fixed header, followed by

(various patterns, some optional %{GREEDYDATA:mailbox_message}|%{GREEDYDATA:mailbox_trace})$

That looks expensive to me if it ends up delivering mailbox_trace. You could try changing that to

"^%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_source} %{SYSLOGPROG} %{GREEDYDATA:mailbox_trace}"

Then use a second grok to see if mailbox_trace can be parsed. In fact I might split it into 3, using the first to grab mailbox_timestamp, mailbox_loglevel, mailbox_protocole, and consuming everything inside the square brackets as a lump. That minimizes the amount of text you are feeding to a pattern that contains alternation and optional fields.

Similarly, looking at the zimbra logs. AMAVIS is an alternation of more than a dozen patterns, most of which start with an AMAVIS_THREAD. So for an AMAVIS_RESULT it will parse that AMAVIS_THREAD a dozen times as it fails to match all the other options in the alternation. That is all wasted work. You could use an alternation of STARTUP, REEXEC and a third pattern that is an AMAVIS_THREAD followed by GREEDYDATA, then grok that GREEDYDATA against an array of anchored patterns.

JeanM · August 16, 2019, 12:49pm

Thanks i change the configuration and now i have only 2-3% CPU utilisation

system · September 13, 2019, 12:49pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.

Topic		Replies	Views
CPU usage for logstash hits over 300% Logstash	6	1113	December 23, 2020
Grok filter help! Logstash	5	526	September 15, 2018
Optimizing Grok patterns Logstash	2	1156	April 11, 2018
Abnormally CPU utilization by Logstash Logstash	4	357	November 22, 2019
How grok filter use more cpu? Logstash	5	571	May 30, 2018

Optimize grok filter

Related topics