Grok filters works separately when combined it does not work. Please help

Please find my conf file. The filters works fine when used separate file but when combined they dont show the parsed fields in kibana.

input {
  file {
    path => "/var/log/messages"
    start_position => "beginning"
	type => "messages"
    start_position => beginning
    sincedb_path => "/dev/null"
    ignore_older => 0
  }
  file {
    path => "/var/log/chef-client.log"
    type => "chef-client"
    start_position => beginning
    sincedb_path => "/dev/null"
    ignore_older => 0
  }  
}
filter {
if [type] == "messages"{
 grok {
                           match => {"message" => "%{SYSLOGTIMESTAMP:logtimeStamp} %{USERNAME:systemname} %{GREEDYDATA:message}"}
                           overwrite => [ "message"]
                           remove_field => ["@timestamp", "@version", "systemname" ]
                         }
						 
mutate {
        split => ["host", "."]
        add_field => { "hostname" => "%{[host][0]}" }
        add_field => { "podName" => "%{[host][1]}" }
        add_field => { "ignore" => "%{[host][2]}" }
        remove_field => ["ignore", "host"]
    }
	mutate {
        split => ["path", "/"]
        add_field => { "logfileName" => "%{[path][3]}" }
        add_field => { "logPath" => "%{[path][2]}" }
        remove_field => ["path", "logPath"]
    }
	
	if [type] == "chef-client" {
                grok {
                           match => {"message" => "\[%{TIMESTAMP_ISO8601:logtimestamp}\] %{SPACE}%{LOGLEVEL:loglevel}:%{SPACE}%{GREEDYDATA:message}"}
                           overwrite => [ "message"]
                           remove_field => ["@timestamp", "@version" ]
                         }
mutate {
        split => ["host", "."]
        add_field => { "hostname" => "%{[host][0]}" }
        add_field => { "podName" => "%{[host][1]}" }
        add_field => { "ignore" => "%{[host][2]}" }
        remove_field => ["ignore", "host"]
    }
mutate {
        split => ["path", "/"]
        add_field => { "logfileName" => "%{[path][3]}" }
        add_field => { "logPath" => "%{[path][2]}" }
        remove_field => ["path", "logPath"]
    }
	
 }
if "_grokparsefailure" in [tags] {
            drop { }
        }
}
}


output { 
  amazon_es {
    hosts => ["${ES_ENDPOINT}"]
    region => "us-west-2"
    index => "test-%{+YYYY.MM.dd}"    	
}
}

In filebeat setting ignore_older to zero disables age based filtering.

In logstash setting ignore_older to zero ignores all files more than zero seconds old, resulting in the file input not consuming any files.

I removed the entry ignore_older => 0. I have different file conf file with pipeline can the index be same for all. pipeline.yml -- /etc/logstash/conf/*.conf. please let me if this is right way and does this work.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.