ES-7.1.1
Kibana-7.1.1
Filebeat 7.0.1
Hey all, I'm trying to ingest the logs from an IIS 7.5 instance directly to ES, and the logs are being sent through successfully. However, every log is giving an error of "Provided Grok expressions do not match field value." (see error below)
I'm glad the logs are being sent but because of this formatting error, I'm unable to properly view the IIS dashboard. I know this has been an issue in the past for others, but I've been unlucky with the solutions offered on other posts. Any input is greatly appreciated!!
(open in new tab to see full pic)
My default.json is bone stock, but I'll post it here just in case.
{
"description": "Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins.",
"processors": [{
"grok": {
"field": "message",
"patterns":[
"%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATHWITHBRACKET:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
"%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}",
"%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} HTTP/%{NUMBER:http.version} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}",
"%{TIMESTAMP_ISO8601:iis.access.time} \[%{IPORHOST:destination.address}\]\(http://%{IPORHOST:destination.address}\) %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} \[%{IPORHOST:source.address}\]\(http://%{IPORHOST:source.address}\) %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
"%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}"
],
"pattern_definitions": {
"URIPATHWITHBRACKET": "(?:/[A-Za-z0-9$.+!'(){},~:;=@#%&_\-\[\]])+"
},
"ignore_missing": true
}
}, {
"remove":{
"field": "message"
}
}, {
"rename": {
"field": "@timestamp",
"target_field": "event.created"
}
}, {
"date": {
"field": "iis.access.time",
"target_field": "@timestamp",
"formats": ["yyyy-MM-dd HH:mm:ss"]
}
}, {
"remove": {
"field": "iis.access.time"
}
}, {
"script": {
"lang": "painless",
"source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)",
"params": { "scale": 1000000 },
"if": "ctx.temp?.duration != null"
}
}, {
"remove": {
"field": "temp.duration",
"ignore_missing": true
}
}, {
"urldecode": {
"field": "user_agent.original"
}
}, {
"user_agent": {
"field": "user_agent.original"
}
}, {
"grok": {
"field": "destination.address",
"ignore_failure": true,
"patterns": [
"%{NOZONEIP:destination.ip}"
],
"pattern_definitions": {
"NOZONEIP": "[^%]"
}
}
}, {
"grok": {
"field": "source.address",
"ignore_failure": true,
"patterns": [
"%{NOZONEIP:source.ip}"
],
"pattern_definitions": {
"NOZONEIP": "[^%]"
}
}
}, {
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
}],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}