After upgrade 5.7 logstash to 6.0 unable to get some logs

After upgrade 5.7 logstash to 6.0 unable to get some logs in kibana

I am getting below error in logstash error log

[2019-11-26T12:17:21,720][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"nne-java-prod-logstash-2019.11", :_type=>"log", :_routing=>nil}, #LogStash::Event:0x2ab724aa], :response=>{"index"=>{"_index"=>"nne-java-prod-logstash-2019.11", "_type"=>"log", "_id"=>"t4ilp24BQ_HGsnDQB4eb", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [timestamp] of type [date] in document with id 't4ilp24BQ_HGsnDQB4eb'", "caused_by"=>{"type"=>"illegal_argument_exception", "reason"=>"Invalid format: "Nov 26 11:48:05""}}}}}

my parsing script below

filter {
if ([fields][wstream] == "rsyslog") {
grok {
match => {
"message" => "(?m)%{SYSLOGTIMESTAMP:timestamp} %{DATA:servername} %{GREEDYDATA:systemmsg}"
}
}
mutate {
strip => ["systemmsg"]
}
if ([systemmsg] =~ "was;") {
grok {
match => {
"systemmsg" => "(?m)%{DATA:appevn};%{DATA:netype};%{DATA:project};%{DATA:env} %{GREEDYDATA:newasmsg}"
}
}

        mutate {
            strip => ["newasmsg"]
        }

        if ([netype] == "sysout") {
            grok {
                match => {
                    "newasmsg" => "(?m)(?<wastimestamp>%{GREEDYDATA:mystimestamp}]) %{DATA:ThreadId} %{DATA:shortName} %{DATA:eventType} %{DATA:className} %{DATA:methodName} %{GREEDYDATA:appmsg}"
                }
            }

            mutate {
                add_field => {
                    "[fields][type]" => "server"
                }
            }
        }

        if ([netype] == "syserr") {
            grok {
                match => {
                    "newasmsg" => "(?m)(?<wastimestamp>%{GREEDYDATA:mystimestamp}]) %{DATA:ThreadId} %{DATA:shortName} %{DATA:eventType} %{DATA:className} %{DATA:methodName} %{GREEDYDATA:appmsg}"
                }
            }

            mutate {
                add_field => {
                    "[fields][type]" => "server"
                }
            }
        }

        # Remove the square bracket in timestamp
        mutate {
            gsub => ["wastimestamp", "\[", ""]
            gsub => ["wastimestamp", "\]", ""]
            add_field => {
                "[fields][env]" => "%{env}"
            }
            add_field => {
                "[fields][project]" => "%{project}"
            }
            add_field => {
                "[fields][wstream]" => "java"
            }
            remove_field => ["systemmsg", "newasmsg", "mystimestamp", "env", "project", "server"]
        }
    }

    if ([systemmsg] =~ "esb;") {
        grok {
            match => {
                "systemmsg" => "(?m)%{DATA:appevn};%{DATA:netype};%{DATA:nenode};%{DATA:env} %{GREEDYDATA:newasmsg}"
            }
        }

        mutate {
            strip => ["newasmsg"]
        }

        if ([netype] == "esblog") {
            grok {
                match => {
                    "newasmsg" => "(?m)%{MONTHDAY:day}-%{MONTHNUM:month}-%{YEAR:year} %{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}#011%{GREEDYDATA:newasmsg}#011%{GREEDYDATA:xmlcontent}"
                }
            }
            mutate {
                add_field => {
                    "timestamp" => "%{year}-%{month}-%{day} %{hour}:%{minute}%{second}"
                }
            }
            date {
                match => ["timestamp", "ISO8601"]
            }
            # Remove the square bracket in timestamp
            mutate {
                add_field => {
                    "[fields][wstream]" => "esb"
                }
                add_field => {
                     "[fields][env]" => "%{env}"
                }

                remove_field => ["newasmsg","year","month","day","hour","minute","second",systemmsg]
            }

        }# end of esblog if loop
    }# end of esb if loop

}

if ([systemmsg] =~ "wps;") {
grok {
patterns_dir => [ "/etc/logstash/patterns.d" ]
match => [ "message", "%{RSYSLOGWPS1}" ]
}

mutate {
  add_field => {
    "[fields][project]" => "wps"
  }

convert => { "finalstatuscode" => "integer" }

convert => { "httpd_responsetime" => "integer" }

convert => { "httpd_bytes" => "integer" }

    convert => { "httpd_response" => "integer" }
    convert => { "httpd_requesttimemilli" => "integer" }
    convert => { "httpd_bytes" => "integer" }

}

if !("_grokparsefailure" in [tags]) {
  mutate {
    remove_field => [ "message","systemmsg",finalstatuscodestr,httpd_bytesstr,httpd_responsetimestr, servername1 ]
  }
}

}
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.