Hi Andrew, thanks for the insights.
I've moved from using CEF integration to "Custom UDP Logs" https://docs.elastic.co/integrations/udp and able to parse most of the fields correctly as shown below.
Few questions:
- Help to confirm if there are any shortcoming or major flaws in the index template or ingest pipeline that are not recommended
- In the custom field
extrahop.detection.description, there are newline \n characters present, which I'm able to remove using processor gsub. Is there a better way there can be processed automatically, and instead a literal newline can be inserted for readability in the logs.
Ingest Pipeline for ExtraHop using Elastic Agent Integration Custom UDP Logs
{
"description": "Pipeline for ExtraHop CEF logs.",
"_meta": {
"package": {
"name": "extrahop"
}
},
"processors": [
{
"set": {
"field": "ecs.version",
"value": "8.9.0"
}
},
{
"rename": {
"field": "message",
"target_field": "event.original",
"ignore_missing": true
}
},
{
"dissect": {
"field": "event.original",
"pattern": "<%{_tmp.header}>%{_tmp.timestamp} %{extrahop.detection.host} CEF:%{cef.version}|%{cef.device.vendor}|%{cef.device.product}|%{cef.device.version}|%{cef.device.event_class_id}|%{cef.name}|%{cef.severity}|%{_tmp.cef_extensions}"
}
},
{
"grok": {
"field": "_tmp.cef_extensions",
"patterns": [
"%{EXTRAHOP_MV_FIELDS:_tmp.mvfields} msg=%{EXTRAHOP_MESSAGE:_tmp.message}"
],
"pattern_definitions": {
"EXTRAHOP_MV_FIELDS": ".*(?= msg=)",
"EXTRAHOP_MESSAGE": "(.*)"
}
}
},
{
"kv": {
"field": "_tmp.mvfields",
"field_split": " ",
"value_split": "=",
"target_field": "_tmp"
}
},
{
"date": {
"field": "_tmp.timestamp",
"formats": [
"ISO8601"
],
"if": "ctx?._tmp?.timestamp != null"
}
},
{
"set": {
"if": "ctx?._tmp?.start != \"none\"",
"field": "extrahop.detection.startTime",
"value": "{{_tmp.start}}"
}
},
{
"set": {
"field": "extrahop.detection.endTime",
"value": "{{_tmp.end}}",
"if": "ctx?._tmp?.end != \"none\""
}
},
{
"set": {
"field": "extrahop.detection.deviceReceiptTime",
"value": "{{_tmp.rt}}",
"if": "ctx?._tmp?.rt != \"none\""
}
},
{
"grok": {
"field": "_tmp.src",
"patterns": [
"%{SRC}"
],
"pattern_definitions": {
"SRC": "(%{IP:source.ip}|%{MAC:source.mac})"
},
"ignore_missing": true
}
},
{
"grok": {
"field": "_tmp.dst",
"patterns": [
"%{DST}"
],
"pattern_definitions": {
"DST": "(%{IP:destination.ip}|%{MAC:destination.mac})"
},
"ignore_missing": true
}
},
{
"geoip": {
"field": "source.ip",
"ignore_missing": true,
"tag": "source geo",
"target_field": "source.geo"
}
},
{
"geoip": {
"field": "destination.ip",
"ignore_missing": true,
"tag": "destination geo",
"target_field": "destination.geo"
}
},
{
"geoip": {
"field": "source.ip",
"database_file": "GeoLite2-ASN.mmdb",
"ignore_missing": true,
"properties": [
"asn",
"organization_name"
],
"target_field": "source.as"
}
},
{
"geoip": {
"field": "destination.ip",
"database_file": "GeoLite2-ASN.mmdb",
"ignore_missing": true,
"properties": [
"asn",
"organization_name"
],
"target_field": "destination.as"
}
},
{
"rename": {
"field": "source.as.asn",
"ignore_missing": true,
"target_field": "source.as.number"
}
},
{
"rename": {
"field": "source.as.organization_name",
"ignore_missing": true,
"target_field": "source.as.organization.name"
}
},
{
"rename": {
"field": "destination.as.asn",
"ignore_missing": true,
"target_field": "destination.as.number"
}
},
{
"rename": {
"field": "destination.as.organization_name",
"ignore_missing": true,
"target_field": "destination.as.organization.name"
}
},
{
"append": {
"if": "ctx?.destination?.ip != null && ctx?.destination?.ip != ''",
"field": "related.ip",
"allow_duplicates": false,
"value": "{{destination.ip}}"
}
},
{
"append": {
"if": "ctx?.destination?.nat?.ip != null && ctx?.destination?.nat?.ip != ''",
"field": "related.ip",
"allow_duplicates": false,
"value": "{{destination.nat.ip}}"
}
},
{
"append": {
"if": "ctx?.source?.ip != null && ctx?.source?.ip != ''",
"field": "related.ip",
"allow_duplicates": false,
"value": "{{source.ip}}"
}
},
{
"append": {
"if": "ctx?.source?.nat?.ip != null && ctx?.source?.nat?.ip != ''",
"field": "related.ip",
"allow_duplicates": false,
"value": "{{source.nat.ip}}"
}
},
{
"gsub": {
"field": "destination.mac",
"ignore_missing": true,
"pattern": "[:.]",
"replacement": "-",
"tag": "gsub mac"
}
},
{
"gsub": {
"field": "source.mac",
"ignore_missing": true,
"pattern": "[:.]",
"replacement": "-",
"tag": "gsub mac"
}
},
{
"uppercase": {
"field": "destination.mac",
"ignore_missing": true
}
},
{
"uppercase": {
"field": "source.mac",
"ignore_missing": true
}
},
{
"set": {
"field": "extrahop.detection.title",
"value": "{{cef.name}}"
}
},
{
"set": {
"field": "extrahop.detection.severity",
"value": "{{cef.severity}}",
"ignore_empty_value": true
}
},
{
"set": {
"field": "extrahop.detection.id",
"value": "{{_tmp.cn1}}"
}
},
{
"set": {
"field": "extrahop.detection.risk_score",
"value": "{{_tmp.cn2}}",
"ignore_empty_value": true
}
},
{
"set": {
"field": "extrahop.detection.url",
"value": "{{_tmp.cs1}}"
}
},
{
"set": {
"field": "extrahop.detection.category",
"value": "{{_tmp.cs2}}"
}
},
{
"set": {
"field": "extrahop.detection.description",
"value": "{{_tmp.message}}"
}
},
{
"gsub": {
"field": "extrahop.detection.description",
"pattern": "\\.\\\\\\\\n",
"replacement": "\\. ",
"ignore_missing": true
}
},
{
"gsub": {
"field": "extrahop.detection.description",
"pattern": "\\\\\\\\n\\*",
"replacement": " \\*",
"ignore_missing": true
}
},
{
"gsub": {
"field": "extrahop.detection.description",
"pattern": "(\\\\\\\\n|\\\\\\\\)",
"replacement": "",
"ignore_missing": true
}
},
{
"remove": {
"field": "event.original",
"ignore_missing": true,
"if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
"ignore_failure": true
}
},
{
"remove": {
"field": [
"cloud",
"host",
"_tmp"
],
"ignore_missing": true
}
}
],
"on_failure": [
{
"remove": {
"field": [
"_tmp"
],
"ignore_missing": true
}
},
{
"append": {
"field": "error.message",
"value": "Processor \"{{ _ingest.on_failure_processor_type }}\" with tag \"{{ _ingest.on_failure_processor_tag }}\" in pipeline \"{{ _ingest.on_failure_pipeline }}\" failed with message \"{{ _ingest.on_failure_message }}\""
}
},
{
"set": {
"field": "event.kind",
"value": "pipeline_error"
}
}
]
}
Index Template
{
"priority": 500,
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "logs"
},
"codec": "best_compression",
"default_pipeline": "logs-extrahop-pipeline-v1.0.0",
"mapping": {
"total_fields": {
"limit": "10000"
},
"ignore_malformed": "true"
},
"query": {
"default_field": [
"input.type",
"log.file.path",
"log.flags",
"log.source.address",
"cef.device.event_class_id",
"cef.device.product",
"cef.device.vendor",
"cef.device.version",
"cef.name",
"cef.severity",
"cef.version",
"destination.ip",
"destination.mac",
"destination.service.name",
"extrahop.detection.category",
"extrahop.detection.description",
"extrahop.detection.deviceReceiptTime",
"extrahop.detection.endTime",
"extrahop.detection.host",
"extrahop.detection.id",
"extrahop.detection.risk_score",
"extrahop.detection.severity",
"extrahop.detection.startTime",
"extrahop.detection.title",
"extrahop.detection.url",
"source.ip",
"source.mac",
"source.service.name",
"related.ip"
]
}
}
},
"mappings": {
"_routing": {
"required": false
},
"numeric_detection": false,
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"_source": {
"excludes": [],
"includes": [],
"enabled": true
},
"dynamic": true,
"dynamic_templates": [],
"date_detection": true,
"properties": {
"extrahop": {
"type": "object",
"properties": {
"detection": {
"type": "object",
"properties": {
"severity": {
"type": "keyword"
},
"risk_score": {
"type": "long"
},
"host": {
"type": "keyword"
},
"description": {
"type": "keyword"
},
"startTime": {
"type": "date"
},
"endTime": {
"type": "date"
},
"id": {
"type": "long"
},
"category": {
"type": "keyword"
},
"deviceReceiptTime": {
"type": "date"
},
"title": {
"type": "keyword"
},
"url": {
"type": "keyword"
}
}
}
}
},
"input": {
"type": "object",
"properties": {
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"@timestamp": {
"ignore_malformed": false,
"type": "date"
},
"related": {
"type": "object",
"properties": {
"ip": {
"type": "ip"
}
}
},
"cef": {
"type": "object",
"properties": {
"severity": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"device": {
"type": "object",
"properties": {
"product": {
"ignore_above": 1024,
"type": "keyword"
},
"event_class_id": {
"ignore_above": 1024,
"type": "keyword"
},
"vendor": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"log": {
"type": "object",
"properties": {
"file": {
"type": "object",
"properties": {
"path": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"offset": {
"type": "long"
},
"flags": {
"ignore_above": 1024,
"type": "keyword"
},
"source": {
"type": "object",
"properties": {
"address": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"data_stream": {
"type": "object",
"properties": {
"namespace": {
"type": "constant_keyword"
},
"type": {
"type": "constant_keyword"
},
"dataset": {
"type": "constant_keyword"
}
}
},
"destination": {
"type": "object",
"properties": {
"service": {
"type": "object",
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ip": {
"type": "ip"
},
"mac": {
"type": "keyword"
}
}
},
"source": {
"type": "object",
"properties": {
"service": {
"type": "object",
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"ip": {
"type": "ip"
},
"mac": {
"type": "keyword"
}
}
},
"event": {
"type": "object",
"properties": {
"module": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"index_options": "docs",
"doc_values": true
},
"dataset": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"index_options": "docs",
"doc_values": true
}
}
}
}
}
},
"index_patterns": [
"logs-extrahop-*"
],
"data_stream": {
"hidden": false,
"allow_custom_routing": false
},
"composed_of": [
".fleet_globals-1",
".fleet_agent_id_verification-1"
],
"_meta": {
"package": {
"name": "extrahop"
}
}
}
