My index contains all geo data but not showing in list while creating Maps from that index

I am performing Geo lookup via Logstash and added geoip filter to get required data. I am getting data in logs but some how it is not getting listed in indexes while configuring layers in Map. Here is my conf file.

input {
  http_poller {
    urls => {
      test2 => {
        method => get
        url => "https://api.cloudflare.com/client/v4/user/audit_logs"
        headers => {
            "X-Auth-Email" => "my@email.com"
            "X-Auth-Key" => "API Key"
            "Content-Type" => "application/json"
        }
     }
    }
	schedule => { cron => "* * * * * UTC"}
	codec => "json"
  }
}
filter {
split
                {
                        field => "result"
                }

mutate		{
			add_field => { "foo_%{somefield}" => "Hello world, from %{[result][actor][email]}" }
		}
geoip
                {
                        source => "[result][actor][ip]"
                }

}
output {
        stdout { codec => rubydebug }
       elasticsearch {
	codec => json
	hosts => [ "myhost:9200" ]
	index => "cloudflare"	
	ssl => true
	ssl_certificate_verification => false
	user => 'elastic'
	password => password
   }
}

And here is geo data I am getting via logs. It is also showing these data in discover section in Kibana.

Can you share you index mapping

GET cloudflare/_mapping

The followinf field should be mapped as geo_point

geoip.location

@ylasri here it is (GET cloudflare/_mapping). I pasted limited fields due to character limit in editor here. Please let me know if you need all fields, I will add in a text file and send.

   {
  "cloudflare" : {
    "mappings" : {
      "properties" : {
        "@timestamp" : {
          "type" : "date"
        },
        "@version" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "foo_%{somefield}" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "geoip" : {
          "properties" : {
            "city_name" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "continent_code" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "country_code2" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "country_code3" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "country_name" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "ip" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "latitude" : {
              "type" : "float"
            },
            "location" : {
              "properties" : {
                "lat" : {
                  "type" : "float"
                },
                "lon" : {
                  "type" : "float"
                }
              }
            },
            "longitude" : {
              "type" : "float"
            },
            "postal_code" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "region_code" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "region_name" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },
            "timezone" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            }
          }
        },
        "host" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "path" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "result" : {
          "properties" : {
            "action" : {
              "properties" : {
                "info" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "result" : {
                  "type" : "boolean"
                },
                "type" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "actor" : {
              "properties" : {
                "email" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "id" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "ip" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "type" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                }
              }
            },
            "id" : {
              "type" : "text",
              "fields" : {
                "keyword" : {
                  "type" : "keyword",
                  "ignore_above" : 256
                }
              }
            },

As said, you need to map correclty the field geoip.location as geo_point

here is an example for your index

PUT cloudflare
{
  "settings": {
    "number_of_replicas": 0,
    "number_of_shards": 1
  },
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "@version": {
        "type": "keyword"
      },
      "geoip": {
        "properties": {
          "city_name": {
            "type": "keyword"
          },
          "continent_code": {
            "type": "keyword"
          },
          "country_code2": {
            "type": "keyword"
          },
          "country_code3": {
            "type": "keyword"
          },
          "country_name": {
            "type": "keyword"
          },
          "ip": {
            "type": "keyword"
          },
          "location": {
            "type": "geo_point"
          },
          "postal_code": {
            "type": "keyword"
          },
          "region_code": {
            "type": "keyword"
          },
          "region_name": {
            "type": "keyword"
          },
          "timezone": {
            "type": "keyword"
          }
        }
      },
      "host": {
        "type": "keyword"
      },
      "path": {
        "type": "keyword"
      },
      "result": {
        "properties": {
          "action": {
            "properties": {
              "info": {
                "type": "keyword"
              },
              "result": {
                "type": "boolean"
              },
              "type": {
                "type": "keyword"
              }
            }
          },
          "actor": {
            "properties": {
              "email": {
                "type": "keyword"
              },
              "id": {
                "type": "keyword"
              },
              "ip": {
                "type": "keyword"
              },
              "type": {
                "type": "keyword"
              }
            }
          },
          "id": {
            "type": "keyword"
          }
        }
      }
    }
  }
}
1 Like

Is there anyway I can put this conversion in logstash filter plugin ?

No, logstash convert mutate filter support only some basic types

You can use logstash to load your default template for the index

what can be the easiest work around for this ?

I deleted my old index and created one with same name with following as I was on working on dev server. And now it works !

PUT cloudflare
{
  "settings": {
    "number_of_replicas": 0,
    "number_of_shards": 1
  },
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "@version": {
        "type": "keyword"
      },
      "geoip": {
        "properties": {
          "city_name": {
            "type": "keyword"
          },
          "continent_code": {
            "type": "keyword"
          },
          "country_code2": {
            "type": "keyword"
          },
          "country_code3": {
            "type": "keyword"
          },
          "country_name": {
            "type": "keyword"
          },
          "ip": {
            "type": "keyword"
          },
          "location": {
            "type": "geo_point"
          },
          "postal_code": {
            "type": "keyword"
          },
          "region_code": {
            "type": "keyword"
          },
          "region_name": {
            "type": "keyword"
          },
          "timezone": {
            "type": "keyword"
          }
        }
      },
      "host": {
        "type": "keyword"
      },
      "path": {
        "type": "keyword"
      },
      "result": {
        "properties": {
          "action": {
            "properties": {
              "info": {
                "type": "keyword"
              },
              "result": {
                "type": "boolean"
              },
              "type": {
                "type": "keyword"
              }
            }
          },
          "actor": {
            "properties": {
              "email": {
                "type": "keyword"
              },
              "id": {
                "type": "keyword"
              },
              "ip": {
                "type": "keyword"
              },
              "type": {
                "type": "keyword"
              }
            }
          },
          "id": {
            "type": "keyword"
          }
        }
      }
    }
  }
}

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.