Here is my complete logstash pipline configuration :
both system and nginx tags are working well for generating the index what I want .
for neither system nor nginx tags, the variables can't be substituted. %{agent}-%{env}-2018.02
#logstash for tlsbeat
input {
tcp {
port => 5045
}
}
filter {
if "_jsonparsefailure" in [tags] {
drop { }
}
json {
source => "message"
}
mutate {
remove_field => ["message"]
add_field => { "[@metadata][index_prefix]" => "%{agent}-%{env}-%{service}" }
remove_field => ["agent","env","host","port","service"]
}
}
output {
if "system" in [tags] {
elasticsearch {
hosts => ["esdatatest01.tls.ad:9200","esdatatest02.tls.ad:9200","esdatatest03.tls.ad:9200","esdatatest04.tls.ad:9200","esdatatest05.tls.ad:9200","esdatatest06.tls.ad:9200","esdatatest07.tls.ad:9200",
"esdatatest08.tls.ad:9200","esdatatest09.tls.ad:9200","esdatatest10.tls.ad:9200","esdatatest11.tls.ad:9200","esdatatest12.tls.ad:9200","esdatatest13.tls.ad:9200","esdatatest14.tls.ad:9200",
"esdatatest15.tls.ad:9200","esdatatest16.tls.ad:9200","esdatatest17.tls.ad:9200","esdatatest18.tls.ad:9200","esdatatest19.tls.ad:9200","esdatatest20.tls.ad:9200"]
template_overwrite => false
manage_template => false
index => "%{[@metadata][index_prefix]}-%{+YYYY.MM}"
sniffing => false
}
}
if "nginx" in [tags] {
elasticsearch {
hosts => ["esdatatest01.tls.ad:9200","esdatatest02.tls.ad:9200","esdatatest03.tls.ad:9200","esdatatest04.tls.ad:9200","esdatatest05.tls.ad:9200","esdatatest06.tls.ad:9200","esdatatest07.tls.ad:9200",
"esdatatest08.tls.ad:9200","esdatatest09.tls.ad:9200","esdatatest10.tls.ad:9200","esdatatest11.tls.ad:9200","esdatatest12.tls.ad:9200","esdatatest13.tls.ad:9200","esdatatest14.tls.ad:9200",
"esdatatest15.tls.ad:9200","esdatatest16.tls.ad:9200","esdatatest17.tls.ad:9200","esdatatest18.tls.ad:9200","esdatatest19.tls.ad:9200","esdatatest20.tls.ad:9200"]
template_overwrite => false
manage_template => false
index => "%{[@metadata][index_prefix]}-%{+YYYY}"
sniffing => false
}
}
if "system" not in [tags] and "nginx" not in [tags] {
elasticsearch {
hosts => ["esdatatest01.tls.ad:9200","esdatatest02.tls.ad:9200","esdatatest03.tls.ad:9200","esdatatest04.tls.ad:9200","esdatatest05.tls.ad:9200","esdatatest06.tls.ad:9200","esdatatest07.tls.ad:9200",
"esdatatest08.tls.ad:9200","esdatatest09.tls.ad:9200","esdatatest10.tls.ad:9200","esdatatest11.tls.ad:9200","esdatatest12.tls.ad:9200","esdatatest13.tls.ad:9200","esdatatest14.tls.ad:9200",
"esdatatest15.tls.ad:9200","esdatatest16.tls.ad:9200","esdatatest17.tls.ad:9200","esdatatest18.tls.ad:9200","esdatatest19.tls.ad:9200","esdatatest20.tls.ad:9200"]
template_overwrite => false
manage_template => false
index => "%{agent}-%{env}-%{+YYYY.MM}"
sniffing => false
}
}
}