Can you import geoip coordinates from outside? Mine are getting reversed


(Danny Patel) #1

Hello,
I'm using ElasticSearch 5.0.1 and logstash 5.0.1. I'm ingesting web traffic logs which also already contain the longitude and latitude coordinates. I would like to use them to create a geoip geohash. Here is my setup details:

Sample Input:
Feb 8 03:07:35 cache-iad2129 www-site-logs[438579]: 71.62.12.143 "-" "-" [08/Feb/2017:03:07:35 +0000] "GET /ios-config.php HTTP/1.1" 200 1002 "(null)" “DataRelease/317 CFNetwork/808.2.16 Darwin/16.3.0" "Charlottesville" "VA" US NA 37.954 -78.358 200 132681 +

Grok Pattern:
%{GREEDYDATA:junk}]: %{IPORHOST:clientip} "%{USER:ident}" "%{USER:auth}" [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|"-") %{QUOTEDSTRING:referrer} %{QUOTEDSTRING:agent} %{QS:geocity} %{QS:georegion} %{WORD:geocountrycode} %{WORD:geocontinentcode} %{NUMBER:geolat} %{NUMBER:geolong} %{NUMBER:origresponse} %{NUMBER:responsetime} %{GREEDYDATA:connstatus}

Logstash Filter Config:
filter {
if [syslogtag] == "www-access" {
mutate {
}

  grok {
    patterns_dir => ['/etc/logstash/patterns']
    add_field => { "logtype" => "fastly" }
    match => {
      "message" => “%{WWW_ACCESS}"
    }
  }
  if "_grokparsefailure" not in [tags] {
    geoip {
      source => "clientip"
      target => "geoip"
      add_field => [ "[geoip][coordinates]", [geolat] ]
      add_field => [ "[geoip][coordinates]", [geolong]]
      add_field => [ "[geoip][longitude]", [geolong]]
      add_field => [ "[geoip][latitude]", [geolat] ]
    }
    mutate {
      convert => [ "[geoip][coordinates]", "float"]
      convert => [ "[geoip][longitude]", "float"]
      convert => [ "[geoip][latitude]", "float"]
    }
    mutate {
      convert => {
        "bytes" => "integer"
        "response" => "integer"
        "responsetime" => "integer"
        "origresponse" => "integer"
      }
      remove_field => ['timestamp']
      remove_field => ['junk']
      gsub => [
        "geocity", "\"", "",
        "georegion", "\"", ""
      ]
    }
    # The connection status when response is completed. Statuses include
    #  X (connection aborted before the response completed),
    #  + (connection may be kept alive after the response is sent),
    #  - (connection will be closed after the response is sent).
    if [connstatus] =~ /\+/ {
      mutate { replace => { "connstatus" => "OPEN"} }
    }
    if [connstatus] =~ /X/ {
      mutate { replace => { "connstatus" => "ABORTED"} }
    }
    if [connstatus] =~ /-/ {
      mutate { replace => { "connstatus" => "CLOSE"} }
    }
  }

}
}

Sample Document in logstash index:
{
"_index": "platform-2017.02.07",
"_type": "platform",
"_id": "AVoWA7wSY5oq8LjuQ-DE",
"_score": 13.083577,
"_source": {
"request": “/ios-config.php”,
"agent": ""DataRelease/317 CFNetwork/808.2.16 Darwin/16.3.0"",
"logstash_host": “redacted.hostname.com",
"auth": "-",
"ident": "-",
"geolong": "-78.358",
"type": "platform",
"connstatus": "OPEN",
"logtype": “www-access”,
"syslogtag": "www-access",
"clientip": "71.62.12.143",
"@version": "1",
"host": “redacted.host.com",
"responsetime": 132681,
"geoip": {
"timezone": "America/New_York",
"ip": "71.62.12.143",
"latitude": 38.098,
"coordinates": [
0,
0
],
"continent_code": "NA",
"city_name": "Charlottesville",
"country_code2": "US",
"country_name": "United States",
"dma_code": 584,
"country_code3": "US",
"region_name": "Virginia",
"location": [
-78.3966,
38.098
],
"postal_code": "22911",
"longitude": -78.3966,
"region_code": "VA"
},
"verb": "GET",
"geocontinentcode": "NA",
"geolat": "37.954",
"geocountrycode": "US",
"message": "Feb 8 03:07:35 cache-iad2129 www-site-logs[438579]: 71.62.12.143 "-" "-" [08/Feb/2017:03:07:35 +0000] "GET /ios-config.php HTTP/1.1" 200 1002 "(null)" “DataRelease/317 CFNetwork/808.2.16 Darwin/16.3.0" "Charlottesville" "VA" US NA 37.954 -78.358 200 132681 +
\n",
"tags": [],
"referrer": ""(null)"",
"@timestamp": "2017-02-07T03:07:35.829Z",
"response": 200,
"bytes": 1002,
"kafka": {
"consumer_group": "logstash-platform",
"partition": 1,
"offset": 764654106,
"topic": "platform",
"key": null
},
"httpversion": "1.1",
"geocity": "Charlottesville",
"rsyslog_timestamp": "2017-02-06T16:40:26.910541-08:00",
"georegion": "VA",
"origresponse": 200
}
},

My problem is that the geiop coordinates are getting added in reverse order, how do you fix that?


(Danny Patel) #2

Here is my template mapping

"logstash": {
"order": 0,
"version": 50001,
"template": "logstash-",
"settings": {
"index": {
"refresh_interval": "5s"
}
},
"mappings": {
"default": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"mapping": {
"norms": false,
"type": "text"
},
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"norms": false,
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"match_mapping_type": "string",
"match": "
"
}
}
],
"_all": {
"norms": false,
"enabled": true
},
"properties": {
"@timestamp": {
"include_in_all": false,
"type": "date"
},
"geoip": {
"dynamic": true,
"properties": {
"ip": {
"type": "ip"
},
"latitude": {
"type": "half_float"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "half_float"
}
}
},
"@version": {
"include_in_all": false,
"type": "keyword"
}
}
}
},
"aliases": {}
},


(Danny Patel) #3


(Danny Patel) #4

Not getting anything on the map


(system) #5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.