Hello Team,
I am using ELK 6.4.0 and Beats (Filebeat, Metricbeat) 6.4.0. My architecture is Beat->logstash->elasticserach->kibana.
I am getting syntax error in logstash file. I have troubleshooted it a lot but no success.
Below is my logstash conf file for input, filter and output:
input {
beats {
port => 5044
ssl => true
ssl_certificate_authorities => ["/etc/pki/tls/ca.crt"]
ssl_certificate => "/etc/pki/tls/server.crt"
ssl_key => "/etc/pki/tls/server.key"
ssl_verify_mode => "peer"
tls_min_version => "1.2"
}
}
filter {
if [type] == "syslog_logs" {
grok {
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
date {
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
else if [type] == "nginx_access" {
grok {
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} \[%{HTTPDATE:[nginx][access][time]}\] \"%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}\" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} \"%{DATA:[nginx][access][referrer]}\" \"%{DATA:[nginx][access][agent]}\""] }
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[nginx][access][time]"
}
useragent {
source => "[nginx][access][agent]"
target => "[nginx][access][user_agent]"
remove_field => "[nginx][access][agent]"
}
geoip {
source => "[nginx][access][remote_ip]"
target => "[nginx][access][geoip]"
}
}
else {
grok {
match => { "message" => [ "\I\,\s\[(?<date-time>[\d\-\w\:\.]+)\s\#(?<pid>\d+)\]\s+(?<loglevel>\w+)\s\-+\s\:\s\[(?<request-id>[\d\w\-]+)\]\s(?<method>[\w\s]+)\s\"(?<path>[\w\/\.]+)\"\s(?<mlp-message>.*)", "\I\,\s\[(?<date-time>[\d\-\w\:\.]+)\s\#(?<pid>[\d]+)\]\s\s(?<loglevel>[\w]+)\s\--\s\:\s\[(?<request-id>[\d\-\w]+)\]\s(?:[cC]urrent\s)?[dD]evice[\s:]+(?<device-id>[\w\s\:]+)", "\w\,\s\[(?<date-time>[\d\-\w\:\.]+)\s\#(?<pid>\d+)\]\s+(?<loglevel>\w+)\s\-+\s\:\s\[(?<request-id>[\d\w\-]+)\]\s(?<mlp-message>.*)", "\w\,\s\[(?<date-time>[\w\-\:\.]+)\s\#(?<pid>\d+)\]\s+(?<loglevel>\w+)\s(?<mlp-message>.*)", "%{COMBINEDAPACHELOG}" ] }
add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_from", "%{host}" ]
}
}
}
output {
if [type] == "syslog_logs"
{
elasticsearch {
hosts => ["xyz:9200"]
sniffing => true
manage_template => false
index => "syslog-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
else if [type] == "nginx_access"
{
elasticsearch {
hosts => ["xyz:9200"]
sniffing => true
manage_template => false
index => "nginxaccess-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
else
{
elasticsearch {
hosts => ["xyz:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
}
Below is the part of my filebeat.yml
- type: log
enabled: true
paths:
- /var/log/syslog
fields_under_root: true
fields:
type: syslog_logs
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
fields_under_root: true
fields:
type: nginx_access
Below is the error when i am trying to restart the logstash service:
[2018-09-28T11:05:52,693][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, => at line 16, column 8 (byte 563) after filter {\nif [type] == \"syslog_logs\" { \n grok {\n match => { \"message\" => [\"%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\\[%{POSINT:[system][syslog][pid]}\\])?: %{GREEDYMULTILINE:[system][syslog][message]}\"] }\n date ", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:41:in `compile_imperative'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:49:in `compile_graph'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:11:in `block in compile_sources'", "org/jruby/RubyArray.java:2486:in `map'", "/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:10:in `compile_sources'", "org/logstash/execution/AbstractPipelineExt.java:157:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:22:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:90:in `initialize'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:38:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:309:in `block in converge_state'"]}
Please help me to troubleshoot the issue.
Thanks in advance.
