Hello ,
I'm beginning with logstash, I need a help to understand why CEF logs sent by security equipment are not parsed correctly.
I am using the filter below but some fields are not named correctly.
here is an sample log
Jul 23 15:07:35 my_host CEF:0|NETSCOUT|Arbor Edge Defense|6.3.0|ATLAS Threat Categories|Blocked Host|7|rt=1563894450000 src=203.0.113.5 cs3Label=Match Type dpt=53 cn2=134 proto=UDP dst=192.0.2.1 cn1=6971382 spt=45292 cs2Label=Protection Group Name cs1Label=IOC Pattern cn1Label=Element Id cn2Label=Protection Group ID cs7Label=Threat Category cs7=Malware cs6=Simda cs1=qekynuq.com cs6Label=Threat Name cs3=dns cs2=Drifts Protection Group
below the pipeline used
indent preformatted text by 4 spaces
input {
udp {
#codec => cef { delimiter => "\r\n"}
port => 10914
type => syslog
tags => ["LOGS-UDP"]
codec => cef
}
}
filter {
# Filter only CEF logs here
if [type] == "CEF" {
# Manipulate the message
mutate {
# Saved the original message into a temporary field
add_field => { "tmp_message" => "%{message}" }
# splits message on the "|" and has index numbers
split => ["message", "|"]
# generate fields for the CEF header
add_field => { "cef_version" => "%{message[0]}" }
add_field => { "cef_device_vendor" => "%{message[1]}" }
add_field => { "cef_device_product" => "%{message[2]}" }
add_field => { "cef_device_version" => "%{message[3]}" }
add_field => { "cef_sig_id" => "%{message[4]}" }
add_field => { "cef_sig_name" => "%{message[5]}" }
add_field => { "cef_sig_severity" => "%{message[6]}" }
}
# Parse the message with field=value formats
kv {
# Note: values with spaces are lost (still getting there)
field_split => " "
trim_key => "<>\[\],"
trim_value => "<>\[\],"
# Only included the fields which are of interest (dont need everything)
include_keys => [ "cat","act","proto","dst","dpt","src","spt","cn1","cs1","cs2","cs3","cs4","cs5","cs6","cs7" ]
}
}
mutate {
# Rename fields to cef_field_names
rename => [ "cat", "cef_traffic_category"]
rename => [ "act", "cef_traffic_action"]
rename => [ "proto", "cef_traffic_proto"]
rename => [ "dst", "cef_traffic_dst_ip"]
rename => [ "dpt", "cef_traffic_dst_port"]
rename => [ "src", "cef_traffic_src_ip"]
rename => [ "spt", "cef_traffic_src_port"]
rename => ["cn1", "Element_ID"]
rename => ["cs1", "IOC_Pattern" ]
rename => ["cs2", "Protection_Group_Name" ]
rename => ["cs3", "Match_Type" ]
rename => ["cs4", "TAXII_Collection_ID" ]
rename => ["cs5", "TAXII_Collection_Title" ]
rename => ["cs6", "Threat_name" ]
rename => ["cs7", "Threat_Category" ]
# Revert original message and remove temporary field
replace => { "message" => "%{tmp_message}" }
remove_field => [ "tmp_message" ]
}
}
}
output {
elasticsearch {
index => "cef-aed%{+YYYY.MM.dd}"
hosts => ["https://es01:9200"]
ssl => true
ssl_certificate_verification => false
cacert => "/usr/share/logstash/certs/ca/ca.crt"
manage_template => true
user => "elastic"
password => "${ELASTIC_PASSWORD}"
}
}
example of output
"transportProtocol" => "UDP",
"host" => "1.2.2.2",
"@version" => "1",
"cefVersion" => "0",
"deviceCustomNumber2" => "280",
"type" => "syslog",
**"deviceEventClassId" => "Filter List",**
"sourceAddress" => "1.1.1.1",
"deviceVendor" => "NETSCOUT",
**"deviceCustom2" => "TEST-PG",**
"@timestamp" => 2021-02-09T14:12:34.731Z,
"destinationPort" => "3389",
"sourcePort" => "47274",
"destinationAddress" => "1.1.1.1",
"severity" => "5",
"deviceReceiptTime" => "1612879950000",
"deviceProduct" => "Arbor Edge Defense",
"name" => "Blocked Host",
"syslog" => "<25>Feb 9 15:12:34 DEVICE-01",
**"deviceCustom2Label" => "Protection Group Name",**
**"deviceCustomNumber2Label" => "Protection Group ID",**
"deviceVersion" => "6.4.1",
"tags" => [[0] "LOGS-UDP"]
As you can see some field dont have the correct name . I was expected to the field formated like below
cn1: Element_ID => deviceCustomNumber1 => Long
cs1: IOC_Pattern => deviceCustomString1 => String
cn2: Protection_Group_ID => deviceCustomNumber2 =>Long
cs2: Protection_Group_Name => deviceCustomString2 => String
cs3: Match_Type => deviceCustomString3 => String
cs4: TAXII_Collection_ID => deviceCustomString4 => String
cs5: TAXII_Collection_Title => deviceCustomString5 => String
cs6: Threat_Name => deviceCustomString6 => String
cs7: Threat_Category => deviceCustomString7 => String
1- I want to be sure that everything will be parsed without issue
2- modify the name of the customs CEF field
Regards
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.