Hi,
I encountered a rather strange situation with message filtering. I've added to my logstash configuration a few log-files: fix_DM****.fix, cluster.log and ms_.log.
For testing purposes I've added for each fix_DMXXX_.fix log filter for keyword "FIX", which is present in every new string of this log-file.
But currently I'm receiving only messages from fix_DM325_.fix log and with keyword "ERROR", while this keyword is not described in filter section for this log.
From all other fix_DM***_.fix logs I'm not receiving at all, while they are growing and contains "FIX" keyword.
I think, logstash successfully reading these files, because sincedb_dm*** files are changing.
Please advise that might be wrong in my configuration. Thanks in advance!
My configuration:
- Logstash v1.5.2
- jdk1.8.0_45-1.8.0_45-fcs
- RedHat 6.3
and my logstash.conf:
input {
file {
path => ["/data/fix_DM317_*.fix"]
type => "DM317FIX-log"
codec => plain { charset => "UTF-8" }
start_position => "end"
sincedb_path => "/var/cache/logstash/sincedb_dm317"
}
file {
path => ["/data/fix_DM325_*.fix"]
type => "DM325FIX-log"
codec => plain { charset => "UTF-8" }
start_position => "end"
sincedb_path => "/var/cache/logstash/sincedb_dm325"
}
file {
path => ["/data/fix_DM399_*.fix"]
type => "DM399FIX-log"
codec => plain { charset => "UTF-8" }
start_position => "end"
sincedb_path => "/var/cache/logstash/sincedb_dm399"
}
file {
path => ["/data/cluster_*.log"]
type => "CLUSTER-log"
codec => plain { charset => "US-ASCII" }
start_position => "end"
sincedb_path => "/var/cache/logstash/sincedb_cluster"
}
file {
path => ["/data/ms_*.log"]
type => "MS-log"
codec => plain { charset => "US-ASCII" }
start_position => "end"
sincedb_path => "/var/cache/logstash/sincedb_ms"
}
}
filter {
if [type] == "DM317FIX-log" and [message] !~ /FIX|PortfolioExposure|MaxDailyAmountSideCumFilter|Internal/ {
drop { }
}
mutate {
add_field => { "[@metadata][zabbix_key_dm317]" => "gl.dm317" }
}
if [type] == "DMC325FIX-log" and [message] !~ /FIX|PortfolioExposure|MaxDailyAmountSideCumFilter|Internal/ {
drop { }
}
mutate {
add_field => { "[@metadata][zabbix_key_dm325]" => "gl.dm325" }
}
if [type] == "DMC399FIX-log" and [message] !~ /FIX|PortfolioExposure|MaxDailyAmountSideCumFilter|Internal/ {
drop { }
}
mutate {
add_field => { "[@metadata][zabbix_key_dm399]" => "gl.dm399" }
}
if [type] == "CLUSTER-log" and [message] =~ /Failed/ or [message] !~ /ERR|fail|cannot/ {
drop { }
}
mutate {
add_field => { "[@metadata][zabbix_key_cluster]" => "gl.cluster" }
}
if [type] == "MS-log" and [message] !~ /Fail|ERROR|Reject|Invalid|Severity|Warning|WARN/ {
drop { }
}
mutate {
add_field => { "[@metadata][zabbix_key_ms]" => "gl.ms" }
}
}
output {
if [type] == "DM317FIX-log"{
zabbix {
zabbix_host => "host"
zabbix_key => "[@metadata][zabbix_key_dm317]"
zabbix_server_host => "10.1.110.71"
zabbix_value => "message"
}
}
if [type] == "DM325FIX-log" {
zabbix {
zabbix_host => "host"
zabbix_key => "[@metadata][zabbix_key_dm325]"
zabbix_server_host => "10.1.110.71"
zabbix_value => "message"
}
}
if [type] == "DM399FIX-log" {
zabbix {
zabbix_host => "host"
zabbix_key => "[@metadata][zabbix_key_dm399]"
zabbix_server_host => "10.1.110.71"
zabbix_value => "message"
}
}
if [type] == "CLUSTER-log" {
zabbix {
zabbix_host => "host"
zabbix_key => "[@metadata][zabbix_key_cluster]"
zabbix_server_host => "10.1.110.71"
zabbix_value => "message"
}
}
if [type] == "MS-log" {
zabbix {
zabbix_host => "host"
zabbix_key => "[@metadata][zabbix_key_ms]"
zabbix_server_host => "10.1.110.71"
zabbix_value => "message"
}
}
}