Hi Team,
I am trying to parse apache access and errors logs via filebeat input prospector, since i have multiple virtual instance configured on the same server like below.
#=========================== Filebeat prospectors =============================
filebeat.prospectors:
- type: log
enabled: true
paths:
- /opt/Logwarhouse/development/first/error_log.*.*
fields:
log_type: apache_error
log_env: dev1
log_instance: first
fields_under_root: true
- type: log
enabled: true
paths:
- /opt/Logwarhouse/development/second/error_log.*.*
fields:
log_type: apache_error
log_env: dev2
log_instance: second
fields_under_root: true
- type: log
paths:
- /opt/Logwarhouse/development/first/access_log*
fields:
log_type: apache_access
log_env: dev1
log_instance: first
fields_under_root: true
exclude_lines: ['a10hm/1.0']
- type: log
paths:
- /opt/Logwarhouse/development/second/access_log*
fields:
log_type: apache_access
log_env: dev2
log_instance: second
fields_under_root: true
exclude_lines: ['a10hm/1.0']
but,when i trying to parse them via logstash i am getting invalid config error.
logstash conf:
input {
beats {
port => 5070
}
}
filter {
if log_type == "apache_access" {
grok {
match => { "message" => "%{NUMBER:responsetime}\s%{IPORHOST:remote_ip}\s-\s%{DATA:user_name}\s\[%{HTTPDATE:timestamp}\]\s%{NUMBER:lastbyte} \"%{WORD:method}\s%{DATA:url}\sHTTP/%{NUMBER:http_version}\"\s%{NUMBER:response_code}\s%{
NUMBER:bytes}( \"%{DATA:referrer}\")?( \"%{DATA:agent}\")?\s%{GREEDYDATA:unwantedline}" }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "timestamp", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "timestamp"
}
useragent {
source => "agent"
target => "user_agent"
remove_field => "agent"
}
}
else if log_type == "apache_error" {
grok {
match => { "message" => ["\[%{APACHE_TIME:[apache][error][timestamp]}\] \[%{LOGLEVEL:[apache][error][level]}\]( \[client %{IPORHOST:[apache][error][client]}\])? %{GREEDYDATA:[apache][error][message]}",
"\[%{APACHE_TIME:[apache][error][timestamp]}\] \[%{DATA:[apache][error][module]}:%{LOGLEVEL:[apache][error][level]}\] \[pid %{NUMBER:[apache][error][pid]}(:tid %{NUMBER:[apache][error][tid]})?\]( \[client %{IPORHOST:[apache][er
ror][client]}\])? %{GREEDYDATA:[apache][error][message1]}" ] }
pattern_definitions => {
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
}
remove_field => "message"
}
mutate {
rename => { "[apache][error][message1]" => "[apache][error][message]" }
}
date {
match => [ "[apache][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
remove_field => "[apache][error][timestamp]"
}
}
}
Error message recieved while testing the config is
[logstash-6.2.4]$ bin/logstash -f apache-test-pipeline.conf --config.test_and_exit
Sending Logstash's logs to /opt/bea/ELKSTACK/logstash-6.2.4/logs which is now configured via log4j2.properties
[2018-11-07T21:07:52,766][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/opt/bea/ELKSTACK/logstash-6.2.4/modules/netflow/configuration"}
[2018-11-07T21:07:52,787][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/opt/bea/ELKSTACK/logstash-6.2.4/modules/fb_apache/configuration"}
[2018-11-07T21:07:53,524][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"arcsight", :directory=>"/opt/bea/ELKSTACK/logstash-6.2.4/vendor/bundle/jruby/2.3.0/gems/x-pack-6.2.4-java/modules/arcsight/configuration"}
[2018-11-07T21:07:54,027][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2018-11-07T21:07:54,618][FATAL][logstash.runner ] The given configuration is invalid. Reason: Expected one of #, ( at line 7, column 15 (byte 66) after filter {
if log_type
[2018-11-07T21:07:54,629][ERROR][org.logstash.Logstash ] java.lang.IllegalStateException: org.jruby.exceptions.RaiseException: (SystemExit) exit