Related question Private ip geoip JSON object / structure and some questions but this is option 2
some other useful resources
- https://cinhtau.net/2018/01/26/enrich-geopoints/
- https://www.elastic.co/assets/bltc44f784ba5a1fc46/logstash-enrichment-slides.pdf
- https://www.elastic.co/blog/elasticsearch-data-enrichment-with-logstash-a-few-security-examples
- Create Custom geoip database for Logstash 5.2
my logstash config
input {
redis {
data_type => "list"
key => "filebeat"
host => "host.local"
}
}
filter {
# For Beat and LSF compatibility
if [beat][hostname] {
if [source] {
if ![file] {
mutate {
add_field => {
"file" => "%{source}"
}
}
}
}
}
## Apache
## https://www.elastic.co/guide/en/logstash/current/logstash-config-for-filebeat-modules.html
if [fileset][module] == "apache2" {
if [fileset][name] == "access" {
grok {
match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
"%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
remove_field => "message"
}
mutate {
add_field => { "read_timestamp" => "%{@timestamp}" }
}
date {
match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
remove_field => "[apache2][access][time]"
}
useragent {
source => "[apache2][access][agent]"
target => "[apache2][access][user_agent]"
remove_field => "[apache2][access][agent]"
}
geoip {
source => "[apache2][access][remote_ip]"
target => "[apache2][access][geoip]"
}
mutate {
add_tag => [ "unique-tag-name" ]
}
}
else if [fileset][name] == "error" {
grok {
match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
"\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
pattern_definitions => {
"APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
}
remove_field => "message"
}
mutate {
rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
}
date {
match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
remove_field => "[apache2][error][timestamp]"
}
}
}
## add geoip data for private ip address
if [apache2][access][remote_ip] {
translate {
exact => true
regex => true
override => true
field => "[apache2][access][remote_ip]"
destination => "geoip"
dictionary_path => "/path/to/mutate/static-ip-geoip-mapping.yml"
fallback => "{\"geoip\":{\"timezone\":\"America/Anchorage\",\"continent_code\":\"NA\",\"country_name\":\"United States\",\"region_code\":\"AK\",\"country_code2\":\"US\",\"country_code3\":\"US\",\"region_name\":\"Alaska\",\"city_name\":\"Anchorage\",\"latitude\":61.19,\"longitude\":-149.8938,\"location\":[61.19,-149.8938]}}"
}
json {
source => "geoip"
target => "[apache2][access][geoip]"
}
}
## add tags for private ip address
if [apache2][access][remote_ip] {
translate {
exact => true
regex => true
override => true
field => "[apache2][access][remote_ip]"
destination => "tags"
dictionary_path => "/path/to/mutate/static-ip-tag-mapping.yml"
fallback => "Unmapped IP Range in the logs, ask networking team for mapping"
}
json {
source => "tags"
add_tag => [ "tags"]
}
}
}
output {
elasticsearch {
hosts => ["host.local:9200"]
ssl => true
index => "logstash-%{+YYYY.MM}"
}
}
static-ip-geoip-mapping.yml
'123.123.123.*': '{"geoip":{"timezone":"America/New_York","continent_code":"NA","country_name":"United States","region_code":"NY","country_code2":"US","country_code3":"US","region_name":"New York","city_name":"New York","latitude":42.5597,"longitude":-83.1138,"location":[42.5597,-83.1138]}}'
'321.321.321.*': '{"geoip":{"timezone":"America/New_York","continent_code":"NA","country_name":"United States","region_code":"NY","country_code2":"US","country_code3":"US","region_name":"New York","city_name":"New York","latitude":42.5597,"longitude":-83.1138,"location":[42.5597,-83.1138]}}'
'111.111.111.*': '{"geoip":{"timezone":"America/New_York","continent_code":"NA","country_name":"United States","region_code":"NY","country_code2":"US","country_code3":"US","region_name":"New York","city_name":"New York","latitude":28.046,"longitude":-82.6848,"location":[28.046,-82.6848]}}'
'888.888.888.*': '{"geoip":{"timezone":"America/New_York","continent_code":"NA","country_name":"United States","region_code":"NY","country_code2":"US","country_code3":"US","region_name":"New York","city_name":"New York","latitude":28.046,"longitude":-82.6848,"location":[28.046,-82.6848]}}'
static-ip-tag-mapping.yml
'123.123.123.*': '{"add_tags":["User Group A","Test1"]}'
'321.321.321.*': '{"add_tags":["User Group B","Test1"]}'
'111.111.111.*': '{"add_tags":["User Group A","Test2"]}'
'888.888.888.*': '{"add_tags":["User Group B","Test3"]}'
the idea is to add the geoip data from the dictionary lookup, then do another dictionary lookup and add a couple of tags as well. I'd like to do this in one dictionary lookup but did not see how to pull and set the data. if the ip match is not found then default to the fallback option (I think this is how fallback is used)
also the field
[apache2][access][geoip]
might exist so I'd like to overwrite it as well