Parse AWS EC2 logs Error

Hi Engineers,
I set up Observability Logs Stream in Kibana for AWS EC2 logs. I have received the aws.ec2_logs, but Message showed "Fail to find message". When I clicked the "View Details" button, the log contents have been parsed correctly. I've no idea why the message cannot be found.
I also checked the Discover. I received an error.message: Provided Grok expressions do not match field value: [account-id action az-id bytes dstaddr dstport end flow-direction instance-id interface-id log-status packets pkt-dst-aws-service pkt-dstaddr pkt-src-aws-service pkt-srcaddr protocol region srcaddr srcport start sublocation-id sublocation-type subnet-id tcp-flags traffic-path type version vpc-id]
Here is my ingest pipeline logs-aws.ec2_logs-1.51.2 Processor configuration:

[
  {
    "set": {
      "field": "ecs.version",
      "value": "8.0.0"
    }
  },
  {
    "rename": {
      "field": "message",
      "target_field": "event.original",
      "ignore_missing": true,
      "if": "ctx.event?.original == null",
      "description": "Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document."
    }
  },
  {
    "remove": {
      "field": "message",
      "ignore_missing": true,
      "if": "ctx.event?.original != null",
      "description": "The `message` field is no longer required if the document has an `event.original` field."
    }
  },
  {
    "grok": {
      "field": "event.original",
      "patterns": [
        "%{NUMBER:account-id} %{NOTSPACE:action} %{NOTSPACE:az-id} %{NUMBER:bytes:int} %{IPORHOST:dstaddr} %{NUMBER:dstport:int} %{NUMBER:end:int} %{NOTSPACE:flow-direction} %{NOTSPACE:instance-id} %{NOTSPACE:interface-id} %{NOTSPACE:log-status} %{NOTSPACE:packets:int} %{NOTSPACE:pkt-dst-aws-service} %{IPORHOST:pkt-dstaddr} %{NOTSPACE:pkt-src-aws-service} %{IPORHOST:pkt-srcaddr} %{NUMBER:protocol:int} %{NOTSPACE:region} %{IPORHOST:srcaddr} %{NUMBER:srcport:int} %{NUMBER:start:int} %{NOTSPACE:sublocation-id} %{NOTSPACE:sublocation-type} %{NOTSPACE:subnet-id} %{NOTSPACE:tcp-flags} %{NOTSPACE:traffic-path} %{NOTSPACE:type} %{NUMBER:version:int} %{NOTSPACE:vpc-id}"
      ]
    }
  },
  {
    "date": {
      "field": "end",
      "formats": [
        "UNIX "
      ],
      "target_field": "endtime",
      "ignore_failure": true
    }
  },
  {
    "date": {
      "field": "start",
      "formats": [
        "UNIX "
      ],
      "target_field": "starttime",
      "ignore_failure": true
    }
  },
   {
    "remove": {
      "field": [
        "_tmp"
      ],
      "ignore_missing": true
    }
  },
  {
    "remove": {
      "field": "event.original",
      "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
      "ignore_failure": true,
      "ignore_missing": true
    }
  },
  {
    "pipeline": {
      "name": "logs-aws.ec2_logs@custom",
      "ignore_missing_pipeline": true
    }
  }
]

Thanks!

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.