Ingest pipeline - format error

Elastic Stack Elasticsearch
1

Hello

I create ingest pipeline, i have an error:

Field [event.created] of type [keyword] doesn't support formats.

My log:
- 10.10.10.11 - [01/Sep/2022:07:48:00 +0100] "- - " 100 - "-" "-"

My ingest:

[
  {
    "pipeline": {
      "if": "ctx.message.startsWith('{')",
      "name": "logs-apache.access-1.3.5-third-party"
    }
  },
  {
    "set": {
      "field": "event.ingested",
      "value": "{{_ingest.timestamp}}"
    }
  },
  {
    "set": {
      "field": "ecs.version",
      "value": "1.12.0"
    }
  },
  {
    "rename": {
      "field": "message",
      "target_field": "event.original"
    }
  },
  {
    "grok": {
      "field": "event.original",
      "patterns": [
                "- %{IPORHOST:source.address}  - \\[%{HTTPDATE:apache.access.time}\\] \"- - \" %{NUMBER:http.response.status_code:long} - \"-\" \"-\""
      ],
      "trace_match": true,
      "ignore_missing": true
    }
  },
  {
    "uri_parts": {
      "field": "_tmp.url_orig",
      "ignore_failure": true
    }
  },
  {
    "remove": {
      "field": [
        "_tmp"
      ],
      "ignore_missing": true
    }
  },
  {
    "set": {
      "field": "url.domain",
      "value": "{{destination.domain}}",
      "if": "ctx.url?.domain == null && ctx.destination?.domain != null"
    }
  },
  {
    "set": {
      "field": "event.kind",
      "value": "event"
    }
  },
  {
    "set": {
      "field": "event.category",
      "value": "web"
    }
  },
  {
    "set": {
      "field": "event.outcome",
      "value": "success",
      "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
    }
  },
  {
    "set": {
      "field": "event.outcome",
      "value": "failure",
      "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
    }
  },
  {
    "grok": {
      "field": "source.address",
      "patterns": [
        "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
      ],
      "trace_match": true,
      "ignore_missing": true
    }
  },
  {
    "remove": {
      "field": "event.created",
      "ignore_missing": true,
      "ignore_failure": true
    }
  },
  {
    "rename": {
      "field": "@timestamp",
      "target_field": "event.created"
    }
  },
  {
    "date": {
      "field": "apache.access.time",
      "target_field": "@timestamp",
      "formats": [
        "dd/MMM/yyyy:H:m:s Z"
      ],
      "ignore_failure": true
    }
  },
  {
    "remove": {
      "field": "apache.access.time",
      "ignore_failure": true
    }
  },
  {
    "user_agent": {
      "field": "user_agent.original",
      "ignore_failure": true
    }
  },
  {
    "geoip": {
      "field": "source.ip",
      "target_field": "source.geo",
      "ignore_missing": true
    }
  },
  {
    "geoip": {
      "database_file": "GeoLite2-ASN.mmdb",
      "field": "source.ip",
      "target_field": "source.as",
      "properties": [
        "asn",
        "organization_name"
      ],
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "field": "source.as.asn",
      "target_field": "source.as.number",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "field": "source.as.organization_name",
      "target_field": "source.as.organization.name",
      "ignore_missing": true
    }
  },
  {
    "set": {
      "field": "tls.cipher",
      "value": "{{apache.access.ssl.cipher}}",
      "if": "ctx?.apache?.access?.ssl?.cipher != null"
    }
  },
  {
    "script": {
      "lang": "painless",
      "if": "ctx?.apache?.access?.ssl?.protocol != null",
      "source": "def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n  return;\n} if (parts[1].contains(\".\")) {\n  ctx.tls.version = parts[1];\n} else {\n  ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"
    }
  },
  {
    "script": {
      "lang": "painless",
      "description": "This script processor iterates over the whole document to remove fields with null values.",
      "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
    }
  },
  {
    "remove": {
      "field": "event.original",
      "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
      "ignore_failure": true,
      "ignore_missing": true
    }
  }
]

What it's wrong?
My policy is a reworked policy apache integration 1.3.5, i changed only grok patterns

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.