File Input is picky!

Elastic Stack Logstash
1

Hey team...Can anyone tell my why my logstash setup is reading one particular log file but not another in the same directory? If I set the path to the working file it starts reading the file and inputing to elasticsearch...change the file name to another file in the same directory and it doesn't read it (but also doesn't error).

input {
  file {
    #path => ["/var/log/remote/**/maillog*.log"] ##This was my attempt at reading the directory - failed.##
    #path => "/var/log/remote/2016/06/08/us11001mx/maillog_00_41.log" ##This file is read and is correctly inputted to elasticsearch##
    path => "/var/log/remote/2016/06/08/us11001mx/maillog_23_50.log" ##This file isn't read, but doesn't error.
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
   stdin {}
}

filter {

  mutate {
    add_field => {
      "message_1" => "%{message}"
      "message_2" => "%{message}"
      "message_3" => "%{message}"
      "message_4" => "%{message}"
      "message_5" => "%{message}"
    }
  }

  grok {
    patterns_dir => "/etc/logstash/patterns/"
    break_on_match => false
    keep_empty_captures => true
    match => [
      "message_1", "(%{SYSLOGBASE} )(%{EXIM_MSGID:exim_msg_id} )(%{EXIM_FLAGS:exim_flags} )(%{GREEDYDATA})"
    ]
    match => [
      "message_2", "(%{EXIM_MSGID} )(<= )(%{NOTSPACE:env_sender} )(%{EXIM_REMOTE_HOST} )?(%{EXIM_INTERFACE} )?(%{EXIM_PROTOCOL} )?(X=%{NOTSPACE:tls_info} )?(%{EXIM_MSG_SIZE} )?(%{EXIM_HEADER_ID} )?(%{EXIM_SUBJECT})"
    ]
    match => [
      "message_3", "(%{EXIM_MSGID} )([=-]> )(%{NOTSPACE:env_rcpt} )(<%{NOTSPACE:env_rcpt_outer}> )?(R=%{NOTSPACE:exim_router} )(T=%{NOTSPACE:exim_transport} )(%{EXIM_REMOTE_HOST} )(X=%{NOTSPACE:tls_info} )?(QT=%{EXIM_QT:exim_qt})"
    ]
    match => [
      "message_4", "(%{SYSLOGBASE} )(%{EXIM_MSGID:exim_msg_id} )(Completed)"
    ]
    match => [
      "message_5", "(%{SYSLOGBASE} )(%{EXIM_MSGID:exim_msg_id} )?(%{EXIM_REMOTE_HOST} )?(%EXIM_INTERFACE} )?(F=<%{NOTSPACE:env_sender}> )?(.+(rejected after DATA|rejected \(but fed to sa-learn\)|rejected [A-Z]+ (or [A-Z]+ %{NOTSPACE}?|<%{NOTSPACE:env_rcpt}>)?): (?<exim_rej_reason>.+))"
    ]
  }


#  if "_grokparsefailure" in [tags] {
#    drop { }
#  }

...
removed mutates for brevity
...

  if "us11-002mrr" in [logsource] {
    mutate {
      add_field => { "host_type" => "Mail Relay" }
    }
  } else if [logsource] =~ /us11-00.mx/ {
    mutate {
      add_field => { "host_type" => "MX" }
    }
  }

  if [exim_msg_state] == "completed" and [host_type] == "MX" {
    elasticsearch {
      query => 'exim_msg_id:"%{exim_msg_id}" AND exim_msg_state:"rejected_after_data"'
      fields => [ "exim_msg_state", "exim_msg_state2" ]
      #sort => "@timestamp:desc, ignore_unmapped:true"
      sort => "ignore_unmapped:true"
      fail_on_error => "false"
    }
    if [exim_msg_state2] == "rejected_after_data" {
      drop { }
    }
    mutate {
      remove_field => [ "query_failed" ]
    }
  }

  if [exim_msg_state] == "delivered" {
    elasticsearch {
      query => 'exim_msg_id:"%{exim_msg_id}" AND exim_msg_state:"received"'
      fields => [ "env_sender", "env_sender", "remote_host", "remote_host", "remote_hostname", "remote_hostname" ]
      sort => "ignore_unmapped:true"
      fail_on_error => "false"
    }
    mutate {
      remove_field => [ "query_failed" ]
    }
  }

  if [exim_flags] == "==" and "retry time not reached" in [message] {
    drop { }
  }

  mutate {
    remove_field => [ "message_1","message_2","message_3","message_4","message_5" ]
  }

}

output {
  elasticsearch {
    hosts => ["192.168.10.170:9200"]
    document_type => "%{[exim_msg_state]}"
    index => "exim-%{+YYYY.MM.dd}"
    flush_size => 2
  }
  stdout {
codec => rubydebug }
}

They're both in the same directory and have the same permissions.

Any ideas?

2

What's the modification time of the files? Have a look at the file input's ignore_older option.