I'm trying to set up filebeat/ELK such that I can drop an apache log file into a directory and it will be indexed automatically.
I'm currently succeeding in having the files indexed, however, each log file is just being indexed at one long string. For example, this is copied from my kibana dashboard:
|Time |message |source |
|March 19th 2018, 12:53:35.172| 66.249.79.132 - - [26/Feb/2018:10:34:58 +0000] "GET /places/businesses/amc/ HTTP/1.0" 404 60 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"| /home/me/apache.log
I'm expecting these logs to be indexed such that each component is an individual field instead of just one long string.
Maybe I'm misunderstanding how to use the grok filter, or there are additional settings I need to include. For example, perhaps I did not properly install the geoip plugin??? I've tried using "%{COMMONAPACHELOG}" and "%{COMBINEDPACHELOG}" but I still get the same results, the log files are indexed as just one string.
Here are my logstash conf files:
02-beats-input.conf
input {
beats {
port => 5044
}
}
10-syslog-filter.conf
filter {
if [type] in [ "apache" , "apache_access" , "apache-access", "syslog", "log" ] {
grok {
match => { "message" => "%{COMMONAPACHELOG}" }
}
mutate {
convert => ["response", "integer"]
convert => ["bytes", "integer"]
convert => ["responsetime", "float"]
}
geoip {
source => "clientip"
target => "geoip"
add_tag => [ "apache-geoip" ]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
remove_field => [ "timestamp" ]
}
useragent {
source => "agent"
}
}
if [type] in ["apache_error","apache-error"] {
grok {
match => ["message", "\[%{WORD:dayname} %{WORD:month} %{DATA:day} %{DATA:hour}:% {DATA:minute}:%{DATA:second} %{YEAR:year}\] \[%{NOTSPACE:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}"]
overwrite => [ "message" ]
}
mutate
{
add_field =>
{
"time_stamp" => "%{day}/%{month}/%{year}:%{hour}:%{minute}:%{second}"
}
}
date {
match => ["time_stamp", "dd/MMM/YYYY:HH:mm:ss"]
remove_field => [ "time_stamp","day","dayname","month","hour","minute","second","year"]
}
}
}
30-elasticsearch-output.conf
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}