Hi, sorry for the massive title. I think that encapsulates what we're seeing. We have a completely new ELK install, with filebeat forwarding directly into elasticsearch. All /var/log/messages entries are failing to parse with "Provided Grok expressions do not match field value:" in the error.message field in ES/Kibana, and none of the fields in the message parsed.
Here's some examples of log messages:
2019-03-29T15:48:37.038843-07:00 redacted sshd[32115]: error: Could not load host key: /etc/ssh/ssh_host_dsa_key
2019-03-29T15:48:50.472292-07:00 redacted su: (to root) ibishop on pts/1
2019-03-29T15:48:58.094462-07:00 redacted rsyslogd: -- MARK --
2019-03-29T15:49:58.153474-07:00 redacted rsyslogd: -- MARK --
2019-03-29T15:50:58.213546-07:00 redacted rsyslogd: -- MARK --
2019-03-29T15:51:10.884977-07:00 redacted iantest: asdflkjsdflkjsdflkj sldfjlsdkjf lsdj f
Same problem with the /var/log/secure logs.
Here's the json of one of the messages:
{
"_index": "filebeat-6.7.0-2019.03.29",
"_type": "doc",
"_id": "NBC1y2kBK6MrnDZw3pgy",
"_version": 1,
"_score": null,
"_source": {
"offset": 910325,
"log": {
"file": {
"path": "/var/log/messages"
}
},
"prospector": {
"type": "log"
},
"source": "/var/log/messages",
"message": "2019-03-29T16:07:59.111429-07:00 redacted rsyslogd: -- MARK --",
"fileset": {
"module": "system",
"name": "syslog"
},
"error": {
"message": "Provided Grok expressions do not match field value: [2019-03-29T16:07:59.111429-07:00 redacted rsyslogd: -- MARK --]"
},
"input": {
"type": "log"
},
"@timestamp": "2019-03-29T23:08:01.571Z",
"beat": {
"hostname": "redacted.redacted.redacted .com",
"name": "redacted.redacted.redacted.com",
"version": "6.7.0"
},
"host": {
"os": {
"codename": "Maipo",
"name": "Red Hat Enterprise Linux Server",
"family": "",
"version": "7.6 (Maipo)",
"platform": "rhel"
},
"containerized": true,
"name": "redacted.redacted.redacted.com",
"id": "21743549ee3f4395870848eb0bf59f29",
"architecture": "x86_64"
},
"event": {
"dataset": "system.syslog"
}
},
"fields": {
"@timestamp": [
"2019-03-29T23:08:01.571Z"
]
},
"sort": [
1553900881571
]
}