Hi @warkolm, Thank you for the reply, and really sorry for replying a bit late. Below is a sample document. It is basically AWS WAF logs pulled from S3 bucket. I have removed sensitive content from the original log line.
{
"timestamp":1600214733254,
"formatVersion":1,
"webaclId":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"terminatingRuleId":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"terminatingRuleType":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"action":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"terminatingRuleMatchDetails":[
],
"httpSourceName":"ALB",
"httpSourceId":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"ruleGroupList":[
{
"ruleGroupId":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"terminatingRule":null,
"nonTerminatingMatchingRules":[
],
"excludedRules":null
},
{
"ruleGroupId":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
"terminatingRule":null,
"nonTerminatingMatchingRules":[
],
"excludedRules":null
}
],
"rateBasedRuleList":[
],
"nonTerminatingMatchingRules":[
],
"httpRequest":{
"clientIp":"196.52.43.125",
"country":"ZA",
"headers":[
{
"name":"Host",
"value":"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
},
{
"name":"User-Agent",
"value":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3602.2 Safari/537.36"
},
{
"name":"Accept",
"value":"*/*"
}
],
"uri":"/",
"args":"",
"httpVersion":"HTTP/1.0",
"httpMethod":"GET",
"requestId":null
}
}
So this document is pulled in by logstash, which comes within the field message. I then convert the message into JSON structure using JSON filter plugin. After that I try to flatten the nested fields among which one field is clientIP. Then I run a geoip filter plugin on this field to get the related data points mentioned in my actual post.