Logstash Date Parse failure

Hello,
I am banging my head for quite sometime to understand why am I getting dateparsefailure after trying multiple match combinations.

Logstash Version = 7.9.2

sudo echo '2021-11-15T11:17:56.831Z' | /usr/share/logstash/bin/logstash -f test.config   
[INFO ] 2021-11-15 12:43:33.912 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9600}
2021
11
15
11
17
56
831/1000
2021-11-15 11:17:56 +0200
{
      "@version" => "1",
          "tags" => [
        [0] "_dateparsefailure"
    ],
    "@timestamp" => 2021-11-15T10:43:33.480Z,
     "timestamp" => 2021-11-15T09:17:56.831Z,
          "host" => "elk-sack-logstash-0",
       "message" => "2021-11-15T11:17:56.831Z"
}

[root@elk-sack-logstash-0 logstash]# cat test.config
input { stdin {} }
output { stdout { codec => rubydebug } }
filter {
  grok {
    match => {
      "message" => "%{TIMESTAMP_ISO8601:timestamp}"
    }
  }
  ruby {
    init => "['date', 'tzinfo'].each(&method(:require))"
    code => "
             tz = TZInfo::Timezone.get('Europe/Helsinki')
             date_obj = DateTime.parse(event.get('timestamp'))
             puts date_obj.year
             puts date_obj.month
             puts date_obj.day
             puts date_obj.hour
             puts date_obj.min
             puts date_obj.sec
             puts date_obj.sec_fraction
             s = tz.local_time(date_obj.year, date_obj.month, date_obj.day, date_obj.hour, date_obj.min, date_obj.sec, date_obj.sec_fraction)
             puts s.inspect
             event.set('timestamp', s)
            "
    }
  date {
    match => ["timestamp", "ISO8601"]
  }
}

I have tried matching timestamp with below formats instead of ISO8601 and each and everyone is throwing error
"yyyy-MM-dd HH:mm:ss,SSSZ"
"yyyy-MM-dd HH:mm:ss.SSSZ"
"yyyy-MM-dd HH:mm:ss,SSS"
"yyyy-MM-dd HH:mm:ss.SSS"
"yyyy-MM-dd HH:mm:ss Z"

Please help

I think you have to respect the structure of your logstash conf file so that :

input {}
filter {}
output {}

Example :

input { stdin {} }

filter {
  grok {  
    match => {
      "message" => "%{TIMESTAMP_ISO8601:timestamp}" 
    }
  }

}

output { stdout { codec => rubydebug } }

@ahmed_charafouddine
Tried with respecting the sequence of config and got the same issue

 sudo echo '2021-04-15T11:17:56.831Z' | /usr/share/logstash/bin/logstash -f test.config
[INFO ] 2021-11-15 16:22:05.861 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9600}
2021
4
15
11
17
56
831/1000
2021-04-15 11:17:56 +0300
{
          "tags" => [
        [0] "_dateparsefailure"
    ],
    "@timestamp" => 2021-11-15T14:22:05.502Z,
      "@version" => "1",
          "host" => "elk-sack-logstash-0",
       "message" => "2021-04-15T11:17:56.831Z",
     "timestamp" => 2021-04-15T08:17:56.831Z
}
[INFO ] 2021-11-15 16:22:07.208 [LogStash::Runner] runner - Logstash shut down.

# cat test.config
input { stdin {} }
filter {
  grok {
    match => {
      "message" => "%{TIMESTAMP_ISO8601:timestamp}"
    }
  }
  ruby {
    init => "['date', 'tzinfo'].each(&method(:require))"
    code => "
             tz = TZInfo::Timezone.get('Europe/Helsinki')
             date_obj = DateTime.parse(event.get('timestamp'))
             puts date_obj.year
             puts date_obj.month
             puts date_obj.day
             puts date_obj.hour
             puts date_obj.min
             puts date_obj.sec
             puts date_obj.sec_fraction
             s = tz.local_time(date_obj.year, date_obj.month, date_obj.day, date_obj.hour, date_obj.min, date_obj.sec, date_obj.sec_fraction)
             puts s.inspect
             event.set('timestamp', s)
            "
    }
  date {
    match => ["timestamp", "ISO8601"]
  }
}
output { stdout { codec => rubydebug } }
 "timestamp" => 2021-11-15T09:17:56.831Z,

There are no quotes around the value, so it is not a string, it is a LogStash::Timestamp object, and the date filter cannot parse them.

It may become clearer if you do not re-use the timestamp field name.

I have no idea how the ruby filter manages to create such an object, but it does.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.