_grokparsefailure and _geoip_lookup_failure when upgrading to ELK 5.0

Elastic Stack Logstash
1

I have installed an ELK Stack log parsing pipeline and I was using ES 2.4, LS 2.4 and Kibana 4, and everything woks fine. Now I am trying to upgrade my Stack into ELK 5.0, unfortunately, I am getting _grokparsefailure and _geoip_lookup_failure on every log line.
Here is my LS config file content:

input {
	file {
		path => "/home/jemlifathi/Documents/dev/training-hub/logs/access_log"
		start_position => "beginning"
		sincedb_path => "/dev/null"
		ignore_older => 0
		type => "logs"
	}
}

filter {
	if[type] == "logs" {
		mutate {
			gsub => ["message", "::ffff:", "",
			 "message", "@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]) \[", " ["]
		}
		grok {
			match=> [
			   "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}",
			   "message" , "%{COMMONAPACHELOG}+%{GREEDYDATA:extra_fields}"
			   ]
			   overwrite=> [ "message" ]
	  	}

	   mutate {
		   convert=> ["response", "integer"]
		   convert=> ["bytes", "integer"]
		   convert=> ["responsetime", "float"]
	   }
	   geoip {
	      source => "clientip"
	      target => "geoip"
	      database => "/etc/logstash/GeoLite2-City.mmdb"
	      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
	      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    	}
	    mutate {
	      convert => [ "[geoip][coordinates]", "float"]
	    }

	   date {
		   match=> [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
		   remove_field=> [ "timestamp" ]
	   }
	   useragent {
	   	   source=> "agent"
	   }
	}
}

output {
	elasticsearch {
		hosts => "localhost:9200"
	}
}
2

Providing some sample data would be helpful :slight_smile:

3

Here is a few lines of my log:

127.0.0.1 - - [20/Feb/2017:17:59:22 +0100] "GET /trainer/node_modules/angular/angular.min.js.map HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
127.0.0.1 - - [20/Feb/2017:17:59:22 +0100] "GET /trainer/node_modules/angular-animate/angular-animate.min.js.map HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
127.0.0.1 - - [20/Feb/2017:17:59:22 +0100] "GET /trainer/node_modules/angular-aria/angular-aria.min.js.map HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
127.0.0.1 - - [20/Feb/2017:17:59:23 +0100] "GET /trainer/assets/layouts/layout4/img/logo-light.png HTTP/1.1" 304 0 "http://training-hub.tn/trainer/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
127.0.0.1 - - [20/Feb/2017:17:59:23 +0100] "GET /trainer/assets/layouts/layout4/img/avatar9.jpg HTTP/1.1" 304 0 "http://training-hub.tn/trainer/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
127.0.0.1 - - [20/Feb/2017:17:59:23 +0100] "GET /trainer/assets/global/plugins/font-awesome/fonts/fontawesome-webfont.woff2?v=4.4.0 HTTP/1.1" 304 0 "http://training-hub.tn/trainer/assets/global/plugins/font-awesome/css/font-awesome.min.css" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
127.0.0.1 - - [20/Feb/2017:17:59:23 +0100] "GET /trainer/assets/layouts/layout4/img/sidebar-toggle-light.png HTTP/1.1" 304 0 "http://training-hub.tn/trainer/assets/layouts/layout4/css/themes/default.min.css" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
::1 - jemlifathi2013@gmail.com [20/Feb/2017:16:59:23 +0000] "GET /api/v1/courses/trainer=58ab00152d3a3c70ae5f3c91 HTTP/1.1" 304 - "http://training-hub.tn/trainer/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.