Hello,
I'm using ElasticSearch 5.0.1 and logstash 5.0.1. I'm ingesting web traffic logs which also already contain the longitude and latitude coordinates. I would like to use them to create a geoip geohash. Here is my setup details:
Sample Input:
Feb 8 03:07:35 cache-iad2129 www-site-logs[438579]: 71.62.12.143 "-" "-" [08/Feb/2017:03:07:35 +0000] "GET /ios-config.php HTTP/1.1" 200 1002 "(null)" “DataRelease/317 CFNetwork/808.2.16 Darwin/16.3.0" "Charlottesville" "VA" US NA 37.954 -78.358 200 132681 +
Grok Pattern:
%{GREEDYDATA:junk}]: %{IPORHOST:clientip} "%{USER:ident}" "%{USER:auth}" [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|"-") %{QUOTEDSTRING:referrer} %{QUOTEDSTRING:agent} %{QS:geocity} %{QS:georegion} %{WORD:geocountrycode} %{WORD:geocontinentcode} %{NUMBER:geolat} %{NUMBER:geolong} %{NUMBER:origresponse} %{NUMBER:responsetime} %{GREEDYDATA:connstatus}
Logstash Filter Config:
filter {
if [syslogtag] == "www-access" {
mutate {
}
grok {
patterns_dir => ['/etc/logstash/patterns']
add_field => { "logtype" => "fastly" }
match => {
"message" => “%{WWW_ACCESS}"
}
}
if "_grokparsefailure" not in [tags] {
geoip {
source => "clientip"
target => "geoip"
add_field => [ "[geoip][coordinates]", [geolat] ]
add_field => [ "[geoip][coordinates]", [geolong]]
add_field => [ "[geoip][longitude]", [geolong]]
add_field => [ "[geoip][latitude]", [geolat] ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
convert => [ "[geoip][longitude]", "float"]
convert => [ "[geoip][latitude]", "float"]
}
mutate {
convert => {
"bytes" => "integer"
"response" => "integer"
"responsetime" => "integer"
"origresponse" => "integer"
}
remove_field => ['timestamp']
remove_field => ['junk']
gsub => [
"geocity", "\"", "",
"georegion", "\"", ""
]
}
# The connection status when response is completed. Statuses include
# X (connection aborted before the response completed),
# + (connection may be kept alive after the response is sent),
# - (connection will be closed after the response is sent).
if [connstatus] =~ /\+/ {
mutate { replace => { "connstatus" => "OPEN"} }
}
if [connstatus] =~ /X/ {
mutate { replace => { "connstatus" => "ABORTED"} }
}
if [connstatus] =~ /-/ {
mutate { replace => { "connstatus" => "CLOSE"} }
}
}
}
}
Sample Document in logstash index:
{
"_index": "platform-2017.02.07",
"_type": "platform",
"_id": "AVoWA7wSY5oq8LjuQ-DE",
"_score": 13.083577,
"_source": {
"request": “/ios-config.php”,
"agent": ""DataRelease/317 CFNetwork/808.2.16 Darwin/16.3.0"",
"logstash_host": “redacted.hostname.com",
"auth": "-",
"ident": "-",
"geolong": "-78.358",
"type": "platform",
"connstatus": "OPEN",
"logtype": “www-access”,
"syslogtag": "www-access",
"clientip": "71.62.12.143",
"@version": "1",
"host": “redacted.host.com",
"responsetime": 132681,
"geoip": {
"timezone": "America/New_York",
"ip": "71.62.12.143",
"latitude": 38.098,
"coordinates": [
0,
0
],
"continent_code": "NA",
"city_name": "Charlottesville",
"country_code2": "US",
"country_name": "United States",
"dma_code": 584,
"country_code3": "US",
"region_name": "Virginia",
"location": [
-78.3966,
38.098
],
"postal_code": "22911",
"longitude": -78.3966,
"region_code": "VA"
},
"verb": "GET",
"geocontinentcode": "NA",
"geolat": "37.954",
"geocountrycode": "US",
"message": "Feb 8 03:07:35 cache-iad2129 www-site-logs[438579]: 71.62.12.143 "-" "-" [08/Feb/2017:03:07:35 +0000] "GET /ios-config.php HTTP/1.1" 200 1002 "(null)" “DataRelease/317 CFNetwork/808.2.16 Darwin/16.3.0" "Charlottesville" "VA" US NA 37.954 -78.358 200 132681 +
\n",
"tags": [],
"referrer": ""(null)"",
"@timestamp": "2017-02-07T03:07:35.829Z",
"response": 200,
"bytes": 1002,
"kafka": {
"consumer_group": "logstash-platform",
"partition": 1,
"offset": 764654106,
"topic": "platform",
"key": null
},
"httpversion": "1.1",
"geocity": "Charlottesville",
"rsyslog_timestamp": "2017-02-06T16:40:26.910541-08:00",
"georegion": "VA",
"origresponse": 200
}
},
My problem is that the geiop coordinates are getting added in reverse order, how do you fix that?
