Kibana 7.8 maps does not recognize logstash index

I have setup ELK stack on Ubuntu. My logstash config ingest my firewall logs. I am using geoip which works fine to locate public IP addresses. "fw" index has been created in Kibana and I can see the events including geoip location, long, lat, etc. from "fw" index in discover, but when I try to create a visualization to show the IP location in a map "fw" index is not in the list to be selected. I only see heartbeat-* and packetbeat-*.

this is my logshtash conf file:

> udp {
>           port => 514
>        }
>        stdin {}
>     }
> 
>     filter {
>       dissect { mapping => { "message" => "<%{loglevel}>%{program}[%{pid}]: %{action} %{[@metadata][restOfLine]}" } }
>        kv { source => "[@metadata][restOfLine]" whitespace => strict
>              exclude_keys => [ "@version", "ACK", "ID", "MARK", "PREC", "SEQ", "TOS", "TTL", "URGP", "WINDOW" ]     }
> 
>        geoip { source => "SRC"
>                target => "src.geo.ip"
>               }
>        geoip { source => "DST"
>                target => "dst.geo.ip"
>               }
>        dns {
>             nameserver => [ "192.168.1.1" ]
>             reverse => [ "dst.geo.ip", "DST" ]
>             reverse => [ "src.geo.ip", "SRC" ]
>             action => "replace"
>             }
>         }
>     output {
>        stdout {}
>        elasticsearch {
>            hosts => ["localhost"]
>            index => "fw"
>        }
>     } 

sample output in discover (Xed out some data for privacy)

> {
>   "_index": "fw",
>   "_type": "_doc",
>   "_id": "xxxxxxxxxxxxxxxxxx6",
>   "_version": 1,
>   "_score": null,
>   "_source": {
>     "MAC": "xxxxxxxxxxxxxxxxxxxxxxxxxxxx",
>     "host": "192.168.1.1",
>     "dst_geoip": {
>       "ip": "xxx.xxx.xxx.xxx",
>       "country_code2": "US",
>       "location": {
>         "lat": xxxxxxxx,
>         "lon": xxxxxxxx
>       },
>       "region_name": "xxxxxxxx",
>       "dma_code": xxx,
>       "country_code3": "US",
>       "continent_code": "NA",
>       "region_code": "xx",
>       "country_name": "United States",
>       "timezone": "America/New_York",
>       "latitude": xxxxxxx,
>       "city_name": "xxxxxxxxxxx",
>       "postal_code": "xxxxxxx",
>       "longitude": xxxxxxxxx
>     },
>     "src_geoip": {
>       "ip": "185.156.73.57",
>       "country_code2": "RU",
>       "continent_code": "EU",
>       "country_name": "Russia",
>       "location": {
>         "lat": 55.7386,
>         "lon": 37.6068
>       },
>       "timezone": "Europe/Moscow",
>       "latitude": 55.7386,
>       "longitude": 37.6068,
>       "country_code3": "RU"
>     },
>     "LEN": "40",
>     "@timestamp": "2020-07-31T02:06:59.598Z",
>     "program": "ulogd",
>     "@version": "1",
>     "pid": "890",
>     "action": "Blocked",
>     "IN": "eth1",
>     "DST": "xxxxxxxxxxxxxxxxxxx",
>     "PROTO": "TCP",
>     "SPT": "49365",
>     "loglevel": "173",
>     "SRC": "185.156.73.57",
>     "DPT": "4928"
>   },
>   "fields": {
>     "@timestamp": [
>       "2020-07-31T02:06:59.598Z"
>     ]
>   },
>   "highlight": {
>     "pid.keyword": [
>       "@kibana-highlighted-field@890@/kibana-highlighted-field@"
>     ],
>     "action.keyword": [
>       "@kibana-highlighted-field@Blocked@/kibana-highlighted-field@"
>     ]
>   },
>   "sort": [
>     12268736219598
>   ]
> }

I am not sure if my logstash conf file is missing something or is incorrect, or something else is missing with my index setup that is not recognized by maps.

Please advice.

Thanks,
Shawn

What is the mapping for the index?

{
  "mappings": {
    "_doc": {
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "@version": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "CODE": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "DPT": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "DST": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "GID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "IN": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "LEN": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "MAC": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "OUT": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "PROTO": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "SPT": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "SRC": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "TYPE": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "UID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "action": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "dst_geoip": {
          "properties": {
            "city_name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "continent_code": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "country_code2": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "country_code3": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "country_name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "dma_code": {
              "type": "long"
            },
            "ip": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "latitude": {
              "type": "float"
            },
            "location": {
              "properties": {
                "lat": {
                  "type": "float"
                },
                "lon": {
                  "type": "float"
                }
              }
            },
            "longitude": {
              "type": "float"
            },
            "postal_code": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "region_code": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "region_name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "timezone": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "host": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "loglevel": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "message": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "pid": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "program": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "src_geoip": {
          "properties": {
            "city_name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "continent_code": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "country_code2": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "country_code3": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "country_name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "dma_code": {
              "type": "long"
            },
            "ip": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "latitude": {
              "type": "float"
            },
            "location": {
              "properties": {
                "lat": {
                  "type": "float"
                },
                "lon": {
                  "type": "float"
                }
              }
            },
            "longitude": {
              "type": "float"
            },
            "postal_code": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "region_code": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "region_name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "timezone": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "tags": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        }
      }
    }
  }
}

Ok, so your location is not mapped as a geo field, that's why.

https://www.elastic.co/blog/geoip-in-the-elastic-stack is an old but relevant post that should help.

You are right Mark, that site is a little old and some commands have changed. I am not sure how can I change the mapping for "fw" index. logstash template has the correct mapping and I have applied that template to "fw" index as well (scressnshot below), but I still cannot see that index when in maps. Is there anyway to modify the index mapping in 7.8?

logstash template mapping:

{
  "_doc": {
    "_meta": {},
    "_source": {},
    "dynamic_templates": [
      {
        "message_field": {
          "path_match": "message",
          "mapping": {
            "norms": false,
            "type": "text"
          },
          "match_mapping_type": "string"
        }
      },
      {
        "string_fields": {
          "mapping": {
            "norms": false,
            "type": "text",
            "fields": {
              "keyword": {
                "ignore_above": 256,
                "type": "keyword"
              }
            }
          },
          "match_mapping_type": "string",
          "match": "*"
        }
      }
    ],
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "geoip": {
        "dynamic": true,
        "type": "object",
        "properties": {
          "ip": {
            "type": "ip"
          },
          "latitude": {
            "type": "half_float"
          },
          "location": {
            "type": "geo_point"
          },
          "longitude": {
            "type": "half_float"
          }
        }
      },
      "@version": {
        "type": "keyword"
      }
    }
  }
}