Hi Engineers,
I set up Observability Logs Stream in Kibana for AWS EC2 logs. I have received the aws.ec2_logs, but Message showed "Fail to find message". When I clicked the "View Details" button, the log contents have been parsed correctly. I've no idea why the message cannot be found.
I also checked the Discover. I received an error.message: Provided Grok expressions do not match field value: [account-id action az-id bytes dstaddr dstport end flow-direction instance-id interface-id log-status packets pkt-dst-aws-service pkt-dstaddr pkt-src-aws-service pkt-srcaddr protocol region srcaddr srcport start sublocation-id sublocation-type subnet-id tcp-flags traffic-path type version vpc-id]
Here is my ingest pipeline logs-aws.ec2_logs-1.51.2 Processor configuration:
[
{
"set": {
"field": "ecs.version",
"value": "8.0.0"
}
},
{
"rename": {
"field": "message",
"target_field": "event.original",
"ignore_missing": true,
"if": "ctx.event?.original == null",
"description": "Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document."
}
},
{
"remove": {
"field": "message",
"ignore_missing": true,
"if": "ctx.event?.original != null",
"description": "The `message` field is no longer required if the document has an `event.original` field."
}
},
{
"grok": {
"field": "event.original",
"patterns": [
"%{NUMBER:account-id} %{NOTSPACE:action} %{NOTSPACE:az-id} %{NUMBER:bytes:int} %{IPORHOST:dstaddr} %{NUMBER:dstport:int} %{NUMBER:end:int} %{NOTSPACE:flow-direction} %{NOTSPACE:instance-id} %{NOTSPACE:interface-id} %{NOTSPACE:log-status} %{NOTSPACE:packets:int} %{NOTSPACE:pkt-dst-aws-service} %{IPORHOST:pkt-dstaddr} %{NOTSPACE:pkt-src-aws-service} %{IPORHOST:pkt-srcaddr} %{NUMBER:protocol:int} %{NOTSPACE:region} %{IPORHOST:srcaddr} %{NUMBER:srcport:int} %{NUMBER:start:int} %{NOTSPACE:sublocation-id} %{NOTSPACE:sublocation-type} %{NOTSPACE:subnet-id} %{NOTSPACE:tcp-flags} %{NOTSPACE:traffic-path} %{NOTSPACE:type} %{NUMBER:version:int} %{NOTSPACE:vpc-id}"
]
}
},
{
"date": {
"field": "end",
"formats": [
"UNIX "
],
"target_field": "endtime",
"ignore_failure": true
}
},
{
"date": {
"field": "start",
"formats": [
"UNIX "
],
"target_field": "starttime",
"ignore_failure": true
}
},
{
"remove": {
"field": [
"_tmp"
],
"ignore_missing": true
}
},
{
"remove": {
"field": "event.original",
"if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"pipeline": {
"name": "logs-aws.ec2_logs@custom",
"ignore_missing_pipeline": true
}
}
]
Thanks!