Logstash filtering issues


(Alexander) #1

Hi,

I encountered a rather strange situation with message filtering. I've added to my logstash configuration a few log-files: fix_DM****.fix, cluster.log and ms_.log.

For testing purposes I've added for each fix_DMXXX_.fix log filter for keyword "FIX", which is present in every new string of this log-file.
But currently I'm receiving only messages from fix_DM325_
.fix log and with keyword "ERROR", while this keyword is not described in filter section for this log.

From all other fix_DM***_.fix logs I'm not receiving at all, while they are growing and contains "FIX" keyword.
I think, logstash successfully reading these files, because sincedb_dm*** files are changing.

Please advise that might be wrong in my configuration. Thanks in advance!

My configuration:

  1. Logstash v1.5.2
  2. jdk1.8.0_45-1.8.0_45-fcs
  3. RedHat 6.3

and my logstash.conf:

input {
  file {
    path => ["/data/fix_DM317_*.fix"]
    type => "DM317FIX-log"
    codec => plain { charset => "UTF-8" }
    start_position => "end"
    sincedb_path => "/var/cache/logstash/sincedb_dm317"
  }
  file {
    path => ["/data/fix_DM325_*.fix"]
    type => "DM325FIX-log"
    codec => plain { charset => "UTF-8" }
    start_position => "end"
    sincedb_path => "/var/cache/logstash/sincedb_dm325"
  }
file {
    path => ["/data/fix_DM399_*.fix"]
    type => "DM399FIX-log"
    codec => plain { charset => "UTF-8" }
    start_position => "end"
    sincedb_path => "/var/cache/logstash/sincedb_dm399"
 }
 file {
    path => ["/data/cluster_*.log"]
    type => "CLUSTER-log"
    codec => plain { charset => "US-ASCII" }
    start_position => "end"
    sincedb_path => "/var/cache/logstash/sincedb_cluster"
 }
 file {
    path => ["/data/ms_*.log"]
    type => "MS-log"
    codec => plain { charset => "US-ASCII" }
    start_position => "end"
    sincedb_path => "/var/cache/logstash/sincedb_ms"
  }
}

filter {
    if [type] == "DM317FIX-log" and [message] !~ /FIX|PortfolioExposure|MaxDailyAmountSideCumFilter|Internal/ {
            drop { }
            }
            mutate {
                    add_field => { "[@metadata][zabbix_key_dm317]" => "gl.dm317" }
            }
    if [type] == "DMC325FIX-log" and [message] !~ /FIX|PortfolioExposure|MaxDailyAmountSideCumFilter|Internal/ {
            drop { }
            }
            mutate {
                    add_field => { "[@metadata][zabbix_key_dm325]" => "gl.dm325" }
            }
    if [type] == "DMC399FIX-log" and [message] !~ /FIX|PortfolioExposure|MaxDailyAmountSideCumFilter|Internal/ {
            drop { }
            }
            mutate {
                    add_field => { "[@metadata][zabbix_key_dm399]" => "gl.dm399" }
            }
    if [type] == "CLUSTER-log" and [message] =~ /Failed/ or [message] !~ /ERR|fail|cannot/ {
            drop { }
            }
            mutate {
                    add_field => { "[@metadata][zabbix_key_cluster]" => "gl.cluster" }
            }
    if [type] == "MS-log" and [message] !~ /Fail|ERROR|Reject|Invalid|Severity|Warning|WARN/ {
            drop { }
            }
            mutate {
                    add_field => { "[@metadata][zabbix_key_ms]" => "gl.ms" }
            }
}

output {
    if [type] == "DM317FIX-log"{
    zabbix {
            zabbix_host => "host"
            zabbix_key => "[@metadata][zabbix_key_dm317]"
            zabbix_server_host => "10.1.110.71"
            zabbix_value => "message"
    }
    }
    if [type] == "DM325FIX-log" {
    zabbix {
            zabbix_host => "host"
            zabbix_key => "[@metadata][zabbix_key_dm325]"
            zabbix_server_host => "10.1.110.71"
            zabbix_value => "message"
    }
    }
    if [type] == "DM399FIX-log" {
    zabbix {
            zabbix_host => "host"
            zabbix_key => "[@metadata][zabbix_key_dm399]"
            zabbix_server_host => "10.1.110.71"
            zabbix_value => "message"
    }
    }
    if [type] == "CLUSTER-log" {
    zabbix {
            zabbix_host => "host"
            zabbix_key => "[@metadata][zabbix_key_cluster]"
            zabbix_server_host => "10.1.110.71"
            zabbix_value => "message"
    }
    }
    if [type] == "MS-log" {
    zabbix {
            zabbix_host => "host"
            zabbix_key => "[@metadata][zabbix_key_ms]"
            zabbix_server_host => "10.1.110.71"
            zabbix_value => "message"
    }
    }
}

(system) #2