Dear all,
i have an issue with logstash that i try to solve since a while. My issue is that the values for the output of "PROBLEMSITE" is always double. Here is an JSON example:
"_source": {
"request": [
"/",
"/"
],
"agent": [
"\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
"\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\""
],
"auth": [
"-",
"-"
],
"ident": [
"-",
"-"
],
"verb": [
"GET",
"GET"
],
All other sites output works as expected!!! No issues, only for "PROBLEMSITE".....
I have a logstash config as follows:
input {
file {
path => [ "/var/log/nginx/access.log" ]
#exclude => "*.gz"
start_position => "beginning"
sincedb_path => "/dev/null"
ignore_older => 0
type => "nginx-default"
}
file {
path => [ "/var/log/nginx/xxx-PROBLEMSITE.access.log" ]
exclude => "*.gz"
start_position => "beginning"
sincedb_path => "/dev/null"
ignore_older => 0
type => "nginx-PROBLEMSITE"
}
file {
path => [ "/var/log/nginx/webmail.access.log" ]
exclude => "*.gz"
start_position => "beginning"
sincedb_path => "/dev/null"
ignore_older => 0
type => "nginx-webmail"
}
file {
path => [ "/var/log/nginx/owncloud.access.log" ]
exclude => "*.gz"
start_position => "beginning"
sincedb_path => "/dev/null"
ignore_older => 0
type => "nginx-owncloud"
}
}
filter {
if [type] == "nginx-default" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "%{COMBINEDAPACHELOG}" ]
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss +0100" ]
target => "tempdate"
}
ruby {
code => "
t = event.get('tempdate').time
t.localtime('+01:00')
event.set('logdatetime', t.strftime('%Y-%m-%dT%H:%M:%S +0100'))
"
}
geoip {
source => "clientip"
target => "geoip-src"
database => "/etc/logstash/GeoLite2-City_20170704/GeoLite2-City.mmdb"
}
}
else if [type] == "nginx-PROBLEMSITE" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "%{COMBINEDAPACHELOG}" ]
}
geoip {
source => "clientip"
target => "geoip-src"
database => "/etc/logstash/GeoLite2-City_20170704/GeoLite2-City.mmdb"
}
}
else if [type] == "nginx-webmail" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "%{ACTIVESYNC}" ]
}
geoip {
source => "clientip"
target => "geoip-src"
database => "/etc/logstash/GeoLite2-City_20170704/GeoLite2-City.mmdb"
}
}
else if [type] == "nginx-owncloud" {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "%{COMBINEDAPACHELOG}" ]
}
geoip {
source => "clientip"
target => "geoip-src"
database => "/etc/logstash/GeoLite2-City_20170704/GeoLite2-City.mmdb"
}
}
}
output {
if [type] == "nginx-default" {
elasticsearch {
index => "nginx-webhoster-default-%{+YYYY-MM-dd}"
hosts => [ "192.168.0.12:9200" ]
#flush_size => 1000
manage_template => true
template_overwrite => true
template => "/etc/logstash/templates/nginx.json"
template_name => "nginx-webhoster-default"
}
}
else if [type] == "nginx-PROBLEMSITE" {
elasticsearch {
index => "nginx-webhoster-PROBLEMSITE-%{+YYYY-MM-dd}"
hosts => [ "192.168.0.12:9200" ]
#flush_size => 1000
manage_template => true
template_overwrite => true
template => "/etc/logstash/templates/nginx.PROBLEMSITE.json"
template_name => "nginx-webhoster-PROBLEMSITE"
}
}
else if [type] == "nginx-webmail" {
elasticsearch {
index => "nginx-webhoster-webmail-%{+YYYY.MM.dd}"
hosts => [ "192.168.0.12:9200" ]
#flush_size => 1000
manage_template => true
template_overwrite => true
template => "/etc/logstash/templates/nginx.webmail.json"
template_name => "nginx-webhoster-webmail"
}
}
else if [type] == "nginx-owncloud" {
elasticsearch {
index => "nginx-webhoster-owncloud-%{+YYYY.MM.dd}"
hosts => [ "192.168.0.12:9200" ]
#flush_size => 1000
manage_template => true
template_overwrite => true
template => "/etc/logstash/templates/nginx.owncloud.json"
template_name => "nginx-webhoster-owncloud"
}
}
}
Could anyone give a hint or shed some light please?