Issue with type geo_pont

Elastic Stack Logstash
1

Hi, I am trying to configure multiple logs in a single conf file. Though all things are coming to kibana but geoip fields are as string rather than geo_point

While configuring separate conf file, geo_point is there and I can see the tile map.
My configuration is as below:

filter {
mutate {
add_field => [ "hostip", "%{host}" ]
}

if [type] == "access" {
grok {
type => "access"
pattern => "%{IPORHOST:remote_addr} - %{USERNAME:remote_user} [%{HTTPDATE:time_local}] "%{WORD:verb} %{URIPATH:app_url}(?:%{URIPARAM:ap_url_param})? HTTP/%{NUMBER:httpversion}" %{INT:status} %{NUMBER:body_bytes_sent:int}"
}

geoip {
source => "remote_addr"
target => "geoip"
database =>"/installdir/ELK/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-1.0.2/vendor/GeoLiteCity-2013-01-18.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}

mutate {
convert => [ "[geoip][coordinates]", "float" ]
}
}
output {
elasticsearch {
cluster => "elasticsearch"
protocol => "http"
action => "index"
host => "192.x.x.x"
index => "%{type}"
template => "/installdir/ELK/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-1.0.7-java/lib/logstash/outputs/elasticsearch/elasticsearch-template.json"
template_name => "access_temp"
workers => 1
}

I have also made changes to template name as:

"template" : "access_temp"

I also tried with if [type] clause in output. still geo_point is not coming.

Can anyone help here

2

Can you link us to the template you are using?

3 
{
  "template" : "access_temp",
  "settings" : {
    "index.refresh_interval" : "5s"
  },
  "mappings" : {
    "_default_" : {
       "_all" : {"enabled" : true, "omit_norms" : true},
       "dynamic_templates" : [ {
         "message_field" : {
           "match" : "message",
           "match_mapping_type" : "string",
           "mapping" : {
             "type" : "string", "index" : "analyzed", "omit_norms" : true
           }
         }
       }, {
         "string_fields" : {
           "match" : "*",
           "match_mapping_type" : "string",
           "mapping" : {
             "type" : "string", "index" : "analyzed", "omit_norms" : true,
               "fields" : {
                 "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
               }
           }
         }
       } ],
       "properties" : {
         "@version": { "type": "string", "index": "not_analyzed" },
         "geoip"  : {
           "type" : "object",
             "dynamic": true,
             "properties" : {
               "location" : { "type" : "geo_point",
                              "lat_lon" : true }
             }
         }
       }
    }
  }
}
4

You're using coordinates, the mapping looks for location.

5

yes I am using lat_lon is anything wrong there?
I have also tried with

output {
    if [type] == "access" {
    elasticsearch {
       cluster => "elasticsearch"
        protocol => "http"
        action => "index"
        host => "x.x.x.x"
        index => "access"
        manage_template => true
        template_overwrite => true
        template => "/installdir/ELK/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-1.0.7-java/lib/logstash/outputs/elasticsearch/elasticsearch-template.json"
        template_name => "access"
        workers => 1
    }
    }
}

and template as below but no luck

   "properties" : {
     "@version": { "type": "string", "index": "not_analyzed" },
     "geoip"  : {
       "type" : "object",
         "dynamic": true,
         "properties" : {
           "location" : { "type" : "geo_point",
                          "lat_lon" : true }
         }
     },
   "location" : { "type": "geo_point" }
   }
6

In your LS config you use [geoip][coordinates].
In the mapping you use geoip.location.

These are not the same field names, the field that is created never matches the mapping. So you need to align the names so they match.

7

I have used lat_lon :true
How come it works when I configure one logstash.conf for one log and not for multiple logs

logstash.conf

output {
    elasticsearch {
       cluster => "elasticsearch"
        protocol => "http"
        action => "index"
        host => "x.x.x.x"
        index => "%{type}"
        template => "/installdir/ELK/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-output-elasticsearch-1.0.7-java/lib/logstash/outputs/elasticsearch/elasticsearch-template-http.json"
        template_name => "ao_http_access"
        workers => 1
    }
     stdout {
         codec => json
     }
}

elasticsearch-template-http.json:

{
  "template" : "ao_http_access",
  "settings" : {
    "index.refresh_interval" : "5s"
  },
  "mappings" : {
    "_default_" : {
       "_all" : {"enabled" : true, "omit_norms" : true},
       "dynamic_templates" : [ {
         "message_field" : {
           "match" : "message",
           "match_mapping_type" : "string",
           "mapping" : {
             "type" : "string", "index" : "analyzed", "omit_norms" : true
           }
         }
       }, {
         "string_fields" : {
           "match" : "*",
           "match_mapping_type" : "string",
           "mapping" : {
             "type" : "string", "index" : "analyzed", "omit_norms" : true,
               "fields" : {
                 "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
               }
           }
         }
       } ],
       "properties" : {
         "@version": { "type": "string", "index": "not_analyzed" },
         "geoip"  : {
           "type" : "object",
             "dynamic": true,
             "properties" : {
               "location" : { "type" : "geo_point",
                              "lat_lon" : true }
             }
         },
       "location" : { "type": "geo_point" }
       }
    }
  }
}
8

I am not sure if you understand what I am saying?

9

If I am doing correct as you are saying, I changed the location to coordinates; still same issue
the geoip.coordinates is now double
the output of localhost:9200/access/_mapping?pretty=true

{
  "ao_access" : {
    "mappings" : {
      "ao_access" : {
        "properties" : {
          "@timestamp" : {
            "type" : "date",
            "format" : "dateOptionalTime"
          },
          "@version" : {
            "type" : "string"
          },
          "ap_url_param" : {
            "type" : "string"
          },
          "app_url" : {
            "type" : "string"
          },
          "body_bytes_sent" : {
            "type" : "long"
          },
          "file" : {
            "type" : "string"
          },
          "geoip" : {
            "properties" : {
              "area_code" : {
                "type" : "long"
              },
              "city_name" : {
                "type" : "string"
              },
              "continent_code" : {
                "type" : "string"
              },
              "coordinates" : {
                "type" : "double"
              },
              "country_code2" : {
                "type" : "string"
              },
              "country_code3" : {
                "type" : "string"
              },
              "country_name" : {
                "type" : "string"
              },
              "dma_code" : {
                "type" : "long"
              },
              "ip" : {
                "type" : "string"
              },
              "latitude" : {
                "type" : "double"
              },
              "location" : {
                "type" : "double"
              },
              "longitude" : {
                "type" : "double"
              },
              "postal_code" : {
                "type" : "string"
              },
              "real_region_name" : {
                "type" : "string"
              },
              "region_name" : {
                "type" : "string"
              },
              "timezone" : {
                "type" : "string"
              }
            }
          },
          "host" : {
            "type" : "string"
          },
          "hostip" : {
            "type" : "string"
          },
          "httpversion" : {
            "type" : "string"
          },
          "message" : {
            "type" : "string"
          },
          "offset" : {
            "type" : "string"
          },
          "remote_addr" : {
            "type" : "string"
          },
          "remote_user" : {
            "type" : "string"
          },
          "status" : {
            "type" : "string"
          },
          "tags" : {
            "type" : "string"
          },
          "time_local" : {
            "type" : "string"
          },
          "type" : {
            "type" : "string"
          },
          "verb" : {
            "type" : "string"
          }
        }
      }
    }
  }
}