Hi @dadoonet, some log with a json codec
For IIS access (type=iis-access)
filter {
if [type] == "iis-access" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:site} %{WORD:method} %{URIPATH:page} %{NOTSPACE:querystring} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clientip} %{NOTSPACE:agent} %{NUMBER:response} %{NUMBER:subresponse} %{NUMBER:scstatus} %{NUMBER:bytes}" }
}
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:http.site} %{NOTSPACE:hostname} %{IPORHOST:locaIp} %{WORD:http.method} %{URIPATH:http.request} %{NOTSPACE:http.msg} %{INT:http.port} %{DATA:username} %{IPORHOST:clientip} %{DATA} %{DATA:http.agent} %{DATA:http.value} %{NUMBER:http.response}" }
}
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:http.method} %{URIPATH:http.request} %{NOTSPACE:http.msg} %{DATA:username} %{IPORHOST:clientip} %{DATA:http.agent} %{DATA:http.value} %{NUMBER:http.response}" }
}
geoip {
source => "clientip"
target => "geoip"
database => "/etc/logstash/GeoLite2-City.mmdb"
}
useragent {
source=> "agent"
target=> "user_agent"
}
useragent {
source=> "http.agent"
target=> "user_agent"
}
translate {
regex => true
dictionary_path => "/etc/logstash/translates/internal-ip.yaml"
field => "clientip"
}
json {
source => "translation"
remove_field => ["translation"]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
mutate {
remove_tag => ["beats_input_codec_plain_applied","_grokparsefailure","_geoip_lookup_failure","_dateparsefailure"]
remove_field => ["log_timestamp","http.msg"]
}
}
}
With public IP, im using GeoIP filter, with internal IP, im using transale with content look like: internal-ip.yaml
'10.1.11.164': '{"geoip": {"unit": "FESFB01", "unit_desc": "FESFB01", "city_name": "Hanoi", "country_name": "Vietnam", "latitude": 21.033, "longitude": 105.85, "location": [105.85, 21.033]}}'
'10.1.11.165': '{"geoip": {"unit": "FESFB02", "unit_desc": "FESFB02", "city_name": "Hanoi", "country_name": "Vietnam", "latitude": 21.033, "longitude": 105.85, "location": [105.85, 21.033]}}'
I think there is a problem with my transate file? but in version 5.x, it worked without problem.