I have below config almost running for 1 year and suddenly i got some open file issues on one of the cluster which i fixed after setting on the limits.conf in linux.
I have below logstash pipeline which is working fine of for the type => "apic_logs"
while it not shiping the logs for type => "noi-syslog"
.
I dont see any errors on the logstash and elasticsearch nodes.
# cat logstash-syslog.conf
input {
file {
path => [ "/scratch/rsyslog/*/messages.log" ]
start_position => beginning
sincedb_path => "/scratch/registry-1"
#sincedb_path => "/dev/null"
max_open_files => 64000
type => "noi-syslog"
}
file {
path => [ "/scratch/rsyslog_CISCO/*/network.log" ]
start_position => beginning
#sincedb_path => "/dev/null"
sincedb_path => "/scratch/registry-2"
max_open_files => 64000
type => "apic_logs"
}
}
filter {
if [type] == "noi-syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
remove_field => [ "host", "path" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
if "_grokparsefailure" in [tags] {
drop { }
}
}
if [type] == "apic_logs" {
grok {
match => { "message" => "%{CISCOTIMESTAMP:syslog_timestamp} %{CISCOTIMESTAMP} %{SYSLOGHOST:syslog_hostname} (?<prog>[\w._/%-]+) %{SYSLOG5424SD:fault_code}%{SYSLOG5424SD:fault_state}%{SYSLOG5424SD:crit_info}%{SYSLOG5424SD:log_severity}%{SYSLOG5424SD:log_info} %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
remove_field => [ "host", "path" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
output {
if [type] == "noi-syslog" {
elasticsearch {
hosts => "host-elk3:9200"
manage_template => false
index => "noi-syslog-%{+YYYY.MM.dd}"
document_type => "messages"
}
# stdout{}
}
}
output {
if [type] == "apic_logs" {
elasticsearch {
hosts => "host-elk2:9200"
manage_template => false
index => "apic_logs-%{+YYYY.MM.dd}"
document_type => "messages"
}
}
}
Please advise any clue or reference in this case.