I have been attempting to parse some WAF logs so they fit with the ECS Schema which I have finally managed to do. However, my final issue is to do with the GeoIP Filter. I wish to utilise the Maps element of Kibana Visualisations and Elastic SIEM Function. The SIEM function states these three errors:
a) unable to find source.geo.location
b) unable to find destination.geo.location
c) cannot read property subtype of undefiend
My Logstash filter is:
filter {
grok {
match => {"message" => "\{\"timestamp\":%{INT:time_of_log},.*" }
}
date {
match => ["time_of_log", "UNIX_MS"]
target => "time_of_log"
}
json {
source => "message"
target => "[message]"
}
mutate {
add_field => { "[ecs][version]" => "1.5" }
add_field => { "json-url-headers" => "[%{[message][httpRequest][headers]}]" }
add_field => { "ruleGrouptList" => "[%{[message][ruleGroupList]}]"}
}
kv {
source => "json-url-headers" field_split_pattern => "\},\{"
value_split_pattern => ","
remove_char_key => "\{\["
trim_key => "name\="
trim_value => "value\="
}
mutate {
rename => { "name=Host" => "Host" }
}
grok {
match => {"[message][webaclId]" => ".\:%{WORD:[cloud][provider]}\:%{WORD:[cloud][instance][name]}\:%{DATA:[cloud][region]}\:%{INT:[cloud][account][id]}"}
}
mutate {
rename => { "[message][action]" => "[event][outcome]" }
rename => { "[message][httpRequest][clientIp]" => "[source][address]" }
rename => { "Host" => "[destination][address]" }
rename => { "User-Agent" => "[user_agent][original]" }
rename => { "[message][httpRequest][httpMethod]" => "[http][request][method]" }
rename => { "[message][httpRequest][httpVersion]" => "[http][version]" }
rename => { "[message][httpRequest][args]" => "[url][query]" }
rename => { "[message][httpRequest][uri]" => "[url][path]" }
rename => { "Referer" => "[http][request][referrer]" }
}
if ("ALLOW" in [event][outcome]){
mutate {
update => { "[event][outcome]" => "success" }
}
}
else if ("BLOCK" in [event][outcome]){
mutate {
update => { "[event][outcome]" => "faliure" }
}
}
grok {
match => {"ruleGroupList" => ".+ruleGroupId\=%{DATA:[rule][ruleset]}[,\}\s]" }
}
geoip {
source => "[source][address]"
target => "[source][geo]"
}
geoip {
source => "[destination][address]"
target => "[destination][geo]"
}
}
I ingest logs in this format:
{
"timestamp": 1576280412771,
"formatVersion": 1,
"webaclId": "arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE",
"terminatingRuleId": "STMTest_SQLi_XSS",
"terminatingRuleType": "REGULAR",
"action": "BLOCK",
"terminatingRuleMatchDetails": [
{
"conditionType": "SQL_INJECTION",
"location": "HEADER",
"matchedData": [
"10",
"AND",
"1"
]
}
],
"httpSourceName": "-",
"httpSourceId": "-",
"ruleGroupList": [],
"rateBasedRuleList": [],
"nonTerminatingMatchingRules": [],
"httpRequest": {
"clientIp": "1.1.1.1",
"country": "AU",
"headers": [
{
"name": "Host",
"value": "localhost:1989"
},
{
"name": "User-Agent",
"value": "curl/7.61.1"
},
{
"name": "Accept",
"value": "*/*"
},
{
"name": "x-stm-test",
"value": "10 AND 1=1"
}
],
"uri": "/foo",
"args": "",
"httpVersion": "HTTP/1.1",
"httpMethod": "GET",
"requestId": "rid"
}
}
Using my logstash filter results in logs in this format. But my issue is the mapping of the geoip fields. As you can see they are assigned as text or number but need to be the type of geopoint I think.