Hi,
I have installed filebeat and enabled the system module, I have also loaded the ingest pipelines manually as the filebeat output is being sent to logstash and to ES from there. The system dashboards were working earlier but have stopped doing so, the message field is not being parsed to JSON. Here are the before(working) and after(not working) JSON messages.
When it was working:
{
"_index": "filebeat-6.6.0-2019.02.19",
"_type": "doc",
"_id": "AChrBGkBWUhMufoBrD1g",
"_version": 1,
"_score": null,
"_source": {
"offset": 67858,
"log": {
"file": {
"path": "/var/log/auth.log"
}
},
"prospector": {
"type": "log"
},
"source": "/var/log/auth.log",
"fileset": {
"module": "system",
"name": "auth"
},
"input": {
"type": "log"
},
"@timestamp": "2019-02-19T06:22:28.000Z",
"system": {
"auth": {
"hostname": "ip-xxxxxxxxxxxx",
"sudo": {
"tty": "pts/1",
"pwd": "/home/ubuntu",
"user": "root",
"command": "/bin/echo Hay! 4.0"
},
"user": "ubuntu",
"timestamp": "Feb 19 06:22:28"
}
},
"meta": {
"cloud": {
"machine_type": "t2.micro",
"availability_zone": "ap-south-1a",
"instance_id": "i-xxxxxxxxxxxxxx",
"provider": "ec2",
"region": "ap-south-1"
}
},
"host": {
"os": {
"codename": "bionic",
"name": "Ubuntu",
"family": "debian",
"version": "18.04.1 LTS (Bionic Beaver)",
"platform": "ubuntu"
},
"containerized": false,
"name": "ip-xxxxxxxxxxxxxx",
"id": "919577b5e29b45cdb2d",
"architecture": "x86_64"
},
"beat": {
"hostname": "ip-xxxxxxxxxxxxxxxxx",
"name": "ip-xxxxxxxxxxxxxxxxx",
"version": "6.6.0"
},
"event": {
"dataset": "system.auth"
}
},
"fields": {
"@timestamp": [
"2019-02-19T06:22:28.000Z"
]
},
"sort": [
1550557348000
]
}
When it stopped working:
{
"_index": "filebeat-6.6.0-2019.02.21",
"_type": "doc",
"_id": "Fd5nD2kBA9gNjQYzof-z",
"_version": 1,
"_score": null,
"_source": {
"message": "Feb 21 09:33:55 ip-172-31-29-94 sudo: root : TTY=pts/1 ; PWD=/var/log ; USER=root ; COMMAND=/bin/echo Hay version 8.0!",
"input": {
"type": "log"
},
"log": {
"file": {
"path": "/var/log/auth.log"
}
},
"beat": {
"name": "ip-xxxxxxxxxxxxx",
"version": "6.6.0",
"hostname": "ip-xxxxxxxxxxxxx"
},
"host": {
"os": {
"name": "Ubuntu",
"version": "18.04.1 LTS (Bionic Beaver)",
"codename": "bionic",
"platform": "ubuntu",
"family": "debian"
},
"name": "ip-xxxxxxxxxxxxx",
"containerized": false,
"id": "919577b5e29b45c360264e490",
"architecture": "x86_64"
},
"tags": [
"beats_input_codec_plain_applied"
],
"@version": "1",
"meta": {
"cloud": {
"instance_id": "i-05014b6e3321",
"machine_type": "t2.micro",
"region": "ap-south-1",
"provider": "ec2",
"availability_zone": "ap-south-1a"
}
},
"fileset": {
"module": "system",
"name": "auth"
},
"event": {
"dataset": "system.auth"
},
"offset": 103306,
"prospector": {
"type": "log"
},
"@timestamp": "2019-02-21T09:34:02.590Z",
"source": "/var/log/auth.log"
},
"fields": {
"@timestamp": [
"2019-02-21T09:34:02.590Z"
]
},
"highlight": {
"message": [
"Feb 21 09:33:55 ip-172-31-29-94 sudo: root : TTY=pts/1 ; PWD=/var/log ; USER=root ; COMMAND=/bin/@kibana-highlighted-field@echo@/kibana-highlighted-field@ Hay version 8.0!"
]
},
"sort": [
1550741642590
]
}
Apart from the message string not being parsed, one of the other differences I see is that the latter JSON has a tag called "beats_input_codec_plain_applied".
Thanks,
Chris