I have setup ELK stack on Ubuntu. My logstash config ingest my firewall logs. I am using geoip which works fine to locate public IP addresses. "fw" index has been created in Kibana and I can see the events including geoip location, long, lat, etc. from "fw" index in discover, but when I try to create a visualization to show the IP location in a map "fw" index is not in the list to be selected. I only see heartbeat-* and packetbeat-*.
this is my logshtash conf file:
> udp {
> port => 514
> }
> stdin {}
> }
>
> filter {
> dissect { mapping => { "message" => "<%{loglevel}>%{program}[%{pid}]: %{action} %{[@metadata][restOfLine]}" } }
> kv { source => "[@metadata][restOfLine]" whitespace => strict
> exclude_keys => [ "@version", "ACK", "ID", "MARK", "PREC", "SEQ", "TOS", "TTL", "URGP", "WINDOW" ] }
>
> geoip { source => "SRC"
> target => "src.geo.ip"
> }
> geoip { source => "DST"
> target => "dst.geo.ip"
> }
> dns {
> nameserver => [ "192.168.1.1" ]
> reverse => [ "dst.geo.ip", "DST" ]
> reverse => [ "src.geo.ip", "SRC" ]
> action => "replace"
> }
> }
> output {
> stdout {}
> elasticsearch {
> hosts => ["localhost"]
> index => "fw"
> }
> }
sample output in discover (Xed out some data for privacy)
> {
> "_index": "fw",
> "_type": "_doc",
> "_id": "xxxxxxxxxxxxxxxxxx6",
> "_version": 1,
> "_score": null,
> "_source": {
> "MAC": "xxxxxxxxxxxxxxxxxxxxxxxxxxxx",
> "host": "192.168.1.1",
> "dst_geoip": {
> "ip": "xxx.xxx.xxx.xxx",
> "country_code2": "US",
> "location": {
> "lat": xxxxxxxx,
> "lon": xxxxxxxx
> },
> "region_name": "xxxxxxxx",
> "dma_code": xxx,
> "country_code3": "US",
> "continent_code": "NA",
> "region_code": "xx",
> "country_name": "United States",
> "timezone": "America/New_York",
> "latitude": xxxxxxx,
> "city_name": "xxxxxxxxxxx",
> "postal_code": "xxxxxxx",
> "longitude": xxxxxxxxx
> },
> "src_geoip": {
> "ip": "185.156.73.57",
> "country_code2": "RU",
> "continent_code": "EU",
> "country_name": "Russia",
> "location": {
> "lat": 55.7386,
> "lon": 37.6068
> },
> "timezone": "Europe/Moscow",
> "latitude": 55.7386,
> "longitude": 37.6068,
> "country_code3": "RU"
> },
> "LEN": "40",
> "@timestamp": "2020-07-31T02:06:59.598Z",
> "program": "ulogd",
> "@version": "1",
> "pid": "890",
> "action": "Blocked",
> "IN": "eth1",
> "DST": "xxxxxxxxxxxxxxxxxxx",
> "PROTO": "TCP",
> "SPT": "49365",
> "loglevel": "173",
> "SRC": "185.156.73.57",
> "DPT": "4928"
> },
> "fields": {
> "@timestamp": [
> "2020-07-31T02:06:59.598Z"
> ]
> },
> "highlight": {
> "pid.keyword": [
> "@kibana-highlighted-field@890@/kibana-highlighted-field@"
> ],
> "action.keyword": [
> "@kibana-highlighted-field@Blocked@/kibana-highlighted-field@"
> ]
> },
> "sort": [
> 12268736219598
> ]
> }
I am not sure if my logstash conf file is missing something or is incorrect, or something else is missing with my index setup that is not recognized by maps.
Please advice.
Thanks,
Shawn
