Illegal pattern component: t when matching date from filebeat system module

lcorsini · July 6, 2017, 8:15am

hi there, I'm using filebeat system module to send logfiles to elastic by using logstash.
I've used the configurations listed here:

https://github.com/elastic/logstash/blob/master/docs/static/filebeat_modules/system/syslog/pipeline.conf

but it's not working on the date filter giving me the

Illegal pattern component: t

(or Illegal pattern component: p on apache2 module)
error.
The wrong line seems to be:
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]

but I don't understand why, given the fact that this seems official logstash information

magnusbaeck · July 9, 2017, 6:58pm

Please show

the exact configuration used (include all files in /etc/logstash/conf.d is that's where you keep your configuration files and
the full error message.

lcorsini · July 10, 2017, 12:05pm

#####FILTER
filter {
grok {
	    match => { 
	        "message" => [
            "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?","%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -",
            "\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}","\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
		"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} from %{IPORHOST:[system][auth][ssh][ip]}",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:[system][auth][groupadd][name]}, GID=%{NUMBER:system.auth.groupadd.gid}",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
            "%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}",
            "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"
	        ] 	
    }
    pattern_definitions => { 
        "GREEDYMULTILINE" => "(.|\n)*" 
        "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" 
    }
}
mutate {
    rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
	    add_field => { "read_timestamp" => "%{@timestamp}" }
}
    if [host] == "stargate" {
    	    date {
           	match => [ "[system][auth][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    		match => [ "[system][syslog][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    	        timezone => "America/New_York"
    	    	#remove_field => ["[system][auth][timestamp]"]
    	    }
    }
    if [host] == "cloud" {
	    date {
            	match => [ "[system][auth][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
		match => [ "[system][syslog][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
		timezone => "Europe/London"
	    	#remove_field => ["[system][auth][timestamp]"]
            }
	 }
    if [host] == "atlantis" {
	    date {
            	match => [ "[system][auth][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
		match => [ "[system][syslog][timestamp]", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
	    	timezone => "Europe/Rome"
	    	#remove_field => ["[system][auth][timestamp]"]
	    }
    }
   
   geoip {
  source => "[system][auth][ssh][ip]"
  target => "[system][auth][ssh][geoip]"
  database => "/usr/local/share/GeoLite2/GeoLite2-City.mmdb"
   }
   geoip {
  source => "[apache2][access][remote_ip]"
  target => "[apache2][access][geoip]"
  database => "/usr/local/share/GeoLite2/GeoLite2-City.mmdb"
   }
   useragent {
  source => "[apache2][access][agent]"
  target => "[apache2][access][user_agent]"
  remove_field => "[apache2][access][agent]"
   }
   if [host] == "cloud" or [host] == "watcher" or [host] == "atlantis" or [host] == "stargate" {
   mutate {
   	       replace => { "host" => "%{host}.lucacorsini.vpn" }
   }
}
}

this is my full filter config, trimmed a bit to post here

lcorsini · July 10, 2017, 12:05pm

and this is the error line:

[2017-07-10T14:00:58,338][ERROR][logstash.agent           ] Cannot create pipeline {:reason=>"Illegal pattern component: t", :backtrace=>["org.joda.time.format.DateTimeFormat.parsePatternTo(org/joda/time/format/DateTimeFormat.java:566)", "org.joda.time.format.DateTimeFormat.createFormatterForPattern(org/joda/time/format/DateTimeFormat.java:687)", "org.joda.time.format.DateTimeFormat.forPattern(org/joda/time/format/DateTimeFormat.java:177)", "org.logstash.filters.parser.JodaParser.<init>(org/logstash/filters/parser/JodaParser.java:58)", "org.logstash.filters.parser.TimestampParserFactory.makeParser(org/logstash/filters/parser/TimestampParserFactory.java:60)", "org.logstash.filters.parser.TimestampParserFactory.makeParser(org/logstash/filters/parser/TimestampParserFactory.java:69)", "org.logstash.filters.DateFilter.acceptFilterConfig(org/logstash/filters/DateFilter.java:66)", "java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:498)", "RUBY.initialize(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-date-3.1.7/lib/logstash/filters/date.rb:185)", "org.jruby.RubyArray.map(org/jruby/RubyArray.java:2414)", "RUBY.initialize(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-date-3.1.7/lib/logstash/filters/date.rb:184)", "RUBY.initialize(/usr/share/logstash/logstash-core/lib/logstash/filter_delegator.rb:21)", "RUBY.plugin(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:107)", "RUBY.initialize((eval):80)", "org.jruby.RubyKernel.eval(org/jruby/RubyKernel.java:1079)", "RUBY.initialize(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:72)", "RUBY.initialize(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:156)", "RUBY.create_pipeline(/usr/share/logstash/logstash-core/lib/logstash/agent.rb:286)", "RUBY.register_pipeline(/usr/share/logstash/logstash-core/lib/logstash/agent.rb:95)", "RUBY.execute(/usr/share/logstash/logstash-core/lib/logstash/runner.rb:314)", "RUBY.run(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/clamp-0.6.5/lib/clamp/command.rb:67)", "RUBY.run(/usr/share/logstash/logstash-core/lib/logstash/runner.rb:209)", "RUBY.run(/usr/share/logstash/vendor/bundle/jruby/1.9/gems/clamp-0.6.5/lib/clamp/command.rb:132)", "usr.share.logstash.lib.bootstrap.environment.(root)(/usr/share/logstash/lib/bootstrap/environment.rb:71)", "usr.share.logstash.lib.bootstrap.environment.(root)(usr/share/logstash/lib/bootstrap//usr/share/logstash/lib/bootstrap/environment.rb:71)"]}

magnusbaeck · July 12, 2017, 10:55am

I'm guessing the problem is the multiple match options in your date filters. They're probably merged in a way where the field name in the second match is interpreted as a date pattern. Split them into multiple filters.

system · August 9, 2017, 10:55am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.