Hey team...Can anyone tell my why my logstash setup is reading one particular log file but not another in the same directory? If I set the path to the working file it starts reading the file and inputing to elasticsearch...change the file name to another file in the same directory and it doesn't read it (but also doesn't error).
input {
file {
#path => ["/var/log/remote/**/maillog*.log"] ##This was my attempt at reading the directory - failed.##
#path => "/var/log/remote/2016/06/08/us11001mx/maillog_00_41.log" ##This file is read and is correctly inputted to elasticsearch##
path => "/var/log/remote/2016/06/08/us11001mx/maillog_23_50.log" ##This file isn't read, but doesn't error.
start_position => "beginning"
sincedb_path => "/dev/null"
}
stdin {}
}
filter {
mutate {
add_field => {
"message_1" => "%{message}"
"message_2" => "%{message}"
"message_3" => "%{message}"
"message_4" => "%{message}"
"message_5" => "%{message}"
}
}
grok {
patterns_dir => "/etc/logstash/patterns/"
break_on_match => false
keep_empty_captures => true
match => [
"message_1", "(%{SYSLOGBASE} )(%{EXIM_MSGID:exim_msg_id} )(%{EXIM_FLAGS:exim_flags} )(%{GREEDYDATA})"
]
match => [
"message_2", "(%{EXIM_MSGID} )(<= )(%{NOTSPACE:env_sender} )(%{EXIM_REMOTE_HOST} )?(%{EXIM_INTERFACE} )?(%{EXIM_PROTOCOL} )?(X=%{NOTSPACE:tls_info} )?(%{EXIM_MSG_SIZE} )?(%{EXIM_HEADER_ID} )?(%{EXIM_SUBJECT})"
]
match => [
"message_3", "(%{EXIM_MSGID} )([=-]> )(%{NOTSPACE:env_rcpt} )(<%{NOTSPACE:env_rcpt_outer}> )?(R=%{NOTSPACE:exim_router} )(T=%{NOTSPACE:exim_transport} )(%{EXIM_REMOTE_HOST} )(X=%{NOTSPACE:tls_info} )?(QT=%{EXIM_QT:exim_qt})"
]
match => [
"message_4", "(%{SYSLOGBASE} )(%{EXIM_MSGID:exim_msg_id} )(Completed)"
]
match => [
"message_5", "(%{SYSLOGBASE} )(%{EXIM_MSGID:exim_msg_id} )?(%{EXIM_REMOTE_HOST} )?(%EXIM_INTERFACE} )?(F=<%{NOTSPACE:env_sender}> )?(.+(rejected after DATA|rejected \(but fed to sa-learn\)|rejected [A-Z]+ (or [A-Z]+ %{NOTSPACE}?|<%{NOTSPACE:env_rcpt}>)?): (?<exim_rej_reason>.+))"
]
}
# if "_grokparsefailure" in [tags] {
# drop { }
# }
...
removed mutates for brevity
...
if "us11-002mrr" in [logsource] {
mutate {
add_field => { "host_type" => "Mail Relay" }
}
} else if [logsource] =~ /us11-00.mx/ {
mutate {
add_field => { "host_type" => "MX" }
}
}
if [exim_msg_state] == "completed" and [host_type] == "MX" {
elasticsearch {
query => 'exim_msg_id:"%{exim_msg_id}" AND exim_msg_state:"rejected_after_data"'
fields => [ "exim_msg_state", "exim_msg_state2" ]
#sort => "@timestamp:desc, ignore_unmapped:true"
sort => "ignore_unmapped:true"
fail_on_error => "false"
}
if [exim_msg_state2] == "rejected_after_data" {
drop { }
}
mutate {
remove_field => [ "query_failed" ]
}
}
if [exim_msg_state] == "delivered" {
elasticsearch {
query => 'exim_msg_id:"%{exim_msg_id}" AND exim_msg_state:"received"'
fields => [ "env_sender", "env_sender", "remote_host", "remote_host", "remote_hostname", "remote_hostname" ]
sort => "ignore_unmapped:true"
fail_on_error => "false"
}
mutate {
remove_field => [ "query_failed" ]
}
}
if [exim_flags] == "==" and "retry time not reached" in [message] {
drop { }
}
mutate {
remove_field => [ "message_1","message_2","message_3","message_4","message_5" ]
}
}
output {
elasticsearch {
hosts => ["192.168.10.170:9200"]
document_type => "%{[exim_msg_state]}"
index => "exim-%{+YYYY.MM.dd}"
flush_size => 2
}
stdout {
codec => rubydebug }
}
They're both in the same directory and have the same permissions.
Any ideas?