Hi all
i'm just very curious to have you input on the below quotation.
find my logstash conf file
input {
beats {
port => "5044"
client_inactivity_timeout => "600"
ssl => false
}
}
filter {
if [fileset][module] == "nginx" {
if [fileset][name] == "access" {
grok {
match => { "message" => ["%{IPORHOST:[nginx][access][remote_ip]} - %{DATA:[nginx][access][user_name]} [%{HTTPDATE:[nginx][access][time]}] "%{WORD:[nginx][access][method]} %{DATA:[nginx][access][url]} HTTP/%{NUMBER:[nginx][access][http_version]}" %{NUMBER:[nginx][access][response_code]} %{NUMBER:[nginx][access][body_sent][bytes]} "%{DATA:[nginx][access][referrer]}" "%{DATA:[nginx][access][agent]}""] }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
convert => [ "[geoip][coordinates]", "float"]
}
date {
match => [ "[nginx][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[nginx][access][time]"
}
useragent {
source => "[nginx][access][agent]"
target => "[nginx][access][user_agent]"
remove_field => "[nginx][access][agent]"
}
geoip {
source => "[nginx][access][remote_ip]"
target => "[geoip]"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
}
if [nginx][access][user_agent][name] =~ ".PingdomBot." {
drop { }
}
if [nginx][access][remote_ip] =~ "172.31.*." {
drop { }
}
else if [fileset][name] == "error" {
grok {
match => { "message" => ["%{DATA:[nginx][error][time]} [%{DATA:[nginx][error][level]}] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
remove_field => "[nginx][error][time]"
}
}
}
}
filter {
if [fileset][module] == "apache2" {
if [fileset][name] == "access" {
if [message] =~ ".ELB-HealthChecker." or [message] =~ ".server-status." {
drop { }
}
grok {
match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} [%{HTTPDATE:[apache2][access][time]}] "%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( "%{DATA:[apache2][access][referrer]}")?( "%{DATA:[apache2][access][agent]}")?",
"%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] "-" %{NUMBER:[apache2][access][response_code]} -" ] }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[apache2][access][time]"
}
useragent {
source => "[apache2][access][agent]"
target => "[apache2][access][user_agent]"
remove_field => "[apache2][access][agent]"
}
geoip {
source => "[apache2][access][remote_ip]"
target => "[geoip]"
}
}
else if [fileset][name] == "error" {
grok {
match => { "message" => ["[%{APACHE_TIME:[apache2][error][timestamp]}] [%{LOGLEVEL:[apache2][error][level]}]( [client %{IPORHOST:[apache2][error][client]}])? %{GREEDYDATA:[apache2][error][message]}",
"[%{APACHE_TIME:[apache2][error][timestamp]}] [%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}] [pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?]( [client %{IPORHOST:[apache2][error][client]}])? %{GREEDYDATA:[apache2][error][message1]}" ] }
pattern_definitions => {
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
}
remove_field => "message"
}
mutate {
rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
}
date {
match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
remove_field => "[apache2][error][timestamp]"
}
}
}
}
filter {
if [fileset][module] == "iis" {
if [fileset][name] == "access" {
if ([message] =~ ".ELB-HealthChecker."){
drop { }
}
grok {
# check that fields match your IIS log settings
match => ["message", "%{TIMESTAMP_ISO8601:logtime} %{IPORHOST:s-ip} %{WORD:cs-method} %{NOTSPACE:cs-uri-stem} %{NOTSPACE:cs-uri-query} %{NUMBER:s-port} %{NOTSPACE:useragent} %{NOTSPACE:sc-status} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:sc-win32-status} %{IPORHOST:OriginalIP}"]
}
date {
match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
timezone => "Etc/UTC"
}
useragent {
source=> "useragent"
prefix=> "browser"
}
geoip {
source => "OriginalIP"
target => "[geoip]"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
#add_field => ["[geoip][location]","%{[geoip][longitude]"]
#add_field => ["[geoip][location]","%{[geoip][latitude]"]
remove_field => [ "log_timestamp"]
remove_field => [ "useragent"]
remove_field => [ "OriginalIP"]
#convert => [ "[geoip][location]", "float" ]
}
}
}
}
output {
if [fileset][module] == "nginx" {
elasticsearch {
hosts => "xxxxxxxxxxxx:443"
ssl => "true"
manage_template => false
index => "nginx-%{+YYYY.MM.dd}"
}
}
if [fileset][module] == "apache2" {
elasticsearch {
hosts => "xxxxxxxxxxxx:443"
ssl => "true"
manage_template => false
index => "apache2-%{+YYYY.MM.dd}"
}
}
if [fields][log_type] == "apache2" {
elasticsearch {
hosts => "xxxxxxxxxxxx:443"
ssl => "true"
manage_template => false
index => "apache2-errors-%{+YYYY.MM.dd}"
}
}
else {
elasticsearch {
hosts => "xxxxxxxxxxxx:443"
ssl => "true"
manage_template => false
index => "test-%{+YYYY.MM.dd}"
}
}
}
my problem
if [fileset][module] == "nginx" {
if [fileset][name] == "access" {
this section works perfectly i can see the index nginx however the
else if [fileset][name] == "error" {
it saved in to "test-%{+YYYY.MM.dd}" index.
its correct that i have multiple filter ?
thanks
M