Filebeat Multiline not working-sundar

Elastic Stack Beats
1

Team,

I am using following filebeat configuration to push magento logs to logstash and from there to kibana. But still its not working as expected . Earlier i used the below logstash configuration without using filebeats and it worked as expected but it eats lots of my server memory .

input {
file {
path => "/var/www/html/var/log/*.log"
path => "/var/log/nginx/access.log"
start_position => "beginning"
codec => multiline {
pattern => "[[\d]{4}"
negate => "true"
what => "previous"
}
sincedb_path => "/dev/null"
}
}

filter {
mutate {
gsub => ["message", "\r|\n", ""]
}
grok {
match => {
"message" => [
#"[%{TIMESTAMP_ISO8601:timestamp}] %{DATA:logger}.%{LOGLEVEL:level}: (?[^{]) %{GREEDYDATA:context}",
"[%{TIMESTAMP_ISO8601:timestamp}] %{DATA:logger}.%{LOGLEVEL:level}: (?[^{]) (?(.|\r|\n)*) ",
"%{GREEDYDATA:logmessage}"
]
}
#match => [ "message", "[%{TIMESTAMP_ISO8601:timestamp}] %{DATA:logger}.%{LOGLEVEL:level}: %{GREEDYDATA:context}", "%{GREEDYDATA:context}"]
}
json {
source => "context"
target => "jsonparsed"
}
}

Now i am trying to push logs via filebeats to logstash My filebeat config and logstash config as below but its not working as expected .

filebeat.inputs:

  • type: log
    enabled: true
    paths:
    • /var/www/html/var/log/restapi.log
      fields:

      used in the output section to send each log to its

      proper index instead of the default 'filebeat-*'

      index_name: qa2magento
      env: qa2magento
      setup.template.enabled: false
      #multiline.pattern: '^[%{TIMESTAMP_ISO8601}]'
      #multiline.negate: true
      #multiline.match: after
      multiline.pattern: '[%{TIMESTAMP_ISO8601}]'
      #multiline.pattern: '^[[:space:]]'
      #multiline.pattern: '^['
      multiline.negate: true
      multiline.match: after

output.logstash:
hosts: ["x.x.x.x:5044"]
bulk_max_size: 1024

index: "%{[fields.index_name]:logs}-%{+YYYY.MM.dd}"

logging:
level: info

logging.to_syslog: false
logging.to_files: true

Logstash config as below

input {
beats {
port => 5044
}
}

filter {
mutate {
gsub => ["message", "\r|\n", ""]
}
grok {
match => {
"message" => [
#"[%{TIMESTAMP_ISO8601:timestamp}] %{DATA:logger}.%{LOGLEVEL:level}: (?[^{]) %{GREEDYDATA:context}",
"[%{TIMESTAMP_ISO8601:timestamp}] %{DATA:logger}.%{LOGLEVEL:level}: (?[^{]) (?(.|\r|\n)*) ",
"%{GREEDYDATA:logmessage}"
]
}
#match => [ "message", "[%{TIMESTAMP_ISO8601:timestamp}] %{DATA:logger}.%{LOGLEVEL:level}: %{GREEDYDATA:context}", "%{GREEDYDATA:context}"]
}
json {
source => "context"
target => "jsonparsed"
}
}

Can someone help me to solve this issue.

Thanks
sundar

2

Could you please format your configuration using </>?

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.