This a totally strange behaviour i am seeing in logstash, I am processing a csv file, where the first column is a date filed, in general when we run the logstash it runs smooth for about 2 - 4 hrs, after which we start getting this error.
{:timestamp=>"2017-08-24T00:53:44.973000-0400", :message=>"Failed parsing date from field",
:field=>"Timestamp", :value=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms",
:exception=>"Invalid format: \"es.e2e_Gx_CCR-I_late_5003.qns_st...\"", :config_parsers=>"yyyy-MM-dd
HH:mm:ss.SSS,ISO8601", :config_locale=>"default=en_US", :level=>:warn}
{:timestamp=>"2017-08-24T00:53:45.679000-0400", :message=>"Failed action. ", :status=>400, :action=>["index", {:_id=>nil, :_index=>"rca_bulkstats_2017.08.24", :_type=>"bulkstats", :_routing=>nil}, #<LogStash::Event:0x6995660d @metadata_accessors=#<LogStash::Util::Accessors:0x7e32b3f3 @store={"path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv"}, @lut={"[path]"=>[{"path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv"}, "path"]}>, @cancelled=false, @data={"message"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms,0\r", "@version"=>"1", "@timestamp"=>"2017-08-24T04:53:43.901Z", "path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv", "host"=>"ubuntu14.04-Node-1", "type"=>"bulkstats", "Timestamp"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms", "SysType"=>"0", "tags"=>["_grokparsefailure", "_dateparsefailure"], "cpu"=>"", "cpu_id"=>"", "cpu_type"=>"", "df_fs"=>"", "df_state"=>"", "domain"=>"", "realm"=>"", "dia_type"=>"", "dia_val"=>"", "dia_status"=>"", "garbage_collector"=>"", "garbage_collector_type"=>"", "set_name"=>"", "num_admindb"=>"", "session_type"=>"", "session_op"=>"", "disk"=>"", "disk_type"=>"", "disk_operation"=>"", "interface"=>"", "Interface_type"=>"", "KPI_Analyzed"=>"%{KPI}", "node"=>""}, @metadata={"path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv"}, @accessors=#<LogStash::Util::Accessors:0x10c178a1 @store={"message"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms,0\r", "@version"=>"1", "@timestamp"=>"2017-08-24T04:53:43.901Z", "path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv", "host"=>"ubuntu14.04-Node-1", "type"=>"bulkstats", "Timestamp"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms", "SysType"=>"0", "tags"=>["_grokparsefailure", "_dateparsefailure"], "cpu"=>"", "cpu_id"=>"", "cpu_type"=>"", "df_fs"=>"", "df_state"=>"", "domain"=>"", "realm"=>"", "dia_type"=>"", "dia_val"=>"", "dia_status"=>"", "garbage_collector"=>"", "garbage_collector_type"=>"", "set_name"=>"", "num_admindb"=>"", "session_type"=>"", "session_op"=>"", "disk"=>"", "disk_type"=>"", "disk_operation"=>"", "interface"=>"", "Interface_type"=>"", "KPI_Analyzed"=>"%{KPI}", "node"=>""}, @lut={"node"=>[{"message"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms,0\r", "@version"=>"1", "@timestamp"=>"2017-08-24T04:53:43.901Z", "path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv", "host"=>"ubuntu14.04-Node-1", "type"=>"bulkstats", "Timestamp"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms", "SysType"=>"0", "tags"=>["_grokparsefailure", "_dateparsefailure"], "cpu"=>"", "cpu_id"=>"", "cpu_type"=>"", "df_fs"=>"", "df_state"=>"", "domain"=>"", "realm"=>"", "dia_type"=>"", "dia_val"=>"", "dia_status"=>"", "garbage_collector"=>"", "garbage_collector_type"=>"", "set_name"=>"", "num_admindb"=>"", "session_type"=>"", "session_op"=>"", "disk"=>"", "disk_type"=>"", "disk_operation"=>"", "interface"=>"", "Interface_type"=>"", "KPI_Analyzed"=>"%{KPI}", "node"=>""}, "node"], "[type]"=>[{"message"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms,0\r", "@version"=>"1", "@timestamp"=>"2017-08-24T04:53:43.901Z", "path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv", "host"=>"ubuntu14.04-Node-1", "type"=>"bulkstats", "Timestamp"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms", "SysType"=>"0", "tags"=>["_grokparsefailure", "_dateparsefailure"], "cpu"=>"", "cpu_id"=>"", "cpu_type"=>"", "df_fs"=>"", "df_state"=>"", "domain"=>"", "realm"=>"", "dia_type"=>"", "dia_val"=>"", "dia_status"=>"", "garbage_collector"=>"", "garbage_collector_type"=>"", "set_name"=>"", "num_admindb"=>"", "session_type"=>"", "session_op"=>"", "disk"=>"", "disk_type"=>"", "disk_operation"=>"", "interface"=>"", "Interface_type"=>"", "KPI_Analyzed"=>"%{KPI}", "node"=>""}, "type"], "[tags]"=>[{"message"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms,0\r", "@version"=>"1", "@timestamp"=>"2017-08-24T04:53:43.901Z", "path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv", "host"=>"ubuntu14.04-Node-1", "type"=>"bulkstats", "Timestamp"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms", "SysType"=>"0", "tags"=>["_grokparsefailure", "_dateparsefailure"], "cpu"=>"", "cpu_id"=>"", "cpu_type"=>"", "df_fs"=>"", "df_state"=>"", "domain"=>"", "realm"=>"", "dia_type"=>"", "dia_val"=>"", "dia_status"=>"", "garbage_collector"=>"", "garbage_collector_type"=>"", "set_name"=>"", "num_admindb"=>"", "session_type"=>"", "session_op"=>"", "disk"=>"", "disk_type"=>"", "disk_operation"=>"", "interface"=>"", "Interface_type"=>"", "KPI_Analyzed"=>"%{KPI}", "node"=>""}, "tags"], "type"=>[{"message"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms,0\r", "@version"=>"1", "@timestamp"=>"2017-08-24T04:53:43.901Z", "path"=>"/mnt/nfs/data/ATT/bulkstats/bulkstats1/agg-stats.6_2594f2cb-7026-49d8-bae4-a50756333b0d.csv", "host"=>"ubuntu14.04-Node-1", "type"=>"bulkstats", "Timestamp"=>"es.e2e_Gx_CCR-I_late_5003.qns_stat.total_time_in_ms", "SysType"=>"0", "tags"=>["_grokparsefailure", "_dateparsefailure"], "cpu"=>"", "cpu_id"=>"", "cpu_type"=>"", "df_fs"=>"", "df_state"=>"", "domain"=>"", "realm"=>"", "dia_type"=>"", "dia_val"=>"", "dia_status"=>"", "garbage_collector"=>"", "garbage_collector_type"=>"", "set_name"=>"", "num_admindb"=>"", "session_type"=>"", "session_op"=>"", "disk"=>"", "disk_type"=>"", "disk_operation"=>"", "interface"=>"", "Interface_type"=>"", "KPI_Analyzed"=>"%{KPI}", "node"=>""}, "type"]}>>], :response=>{"create"=>{"_index"=>"rca_bulkstats_2017.08.24", "_type"=>"bulkstats", "_id"=>"AV4SlsiHUaLbJFtf9MDB", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse [Timestamp]", "caused_by"=>{"type"=>"illegal_argument_exception", "reason"=>"Invalid format: \"es.e2e_Gx_CCR-I_late_5003.qns_st...\""}}}}, :level=>:warn}
When we take the file which got this error and reprocess it, it works smooth.
So we are not able to figure out what happens after 2 - 4 hrs that this strange behaviour starts happening? Checked the cpu and memory both looks good as in not under pressure.
Need help !