Hello,
I got filebeat to feed apache access and error log to elasticsearch. It is awesome and the filebeat provided dashboard is great as well.
However, when I filtered and show only data from the error log, the map will become empty. There are about 5000 records available after the filtering and most of them got geo_point field data available. Could anyone shed the light for me?
below are the data example for your information:
# this is data from error log
{
"_index": "filebeat-7.5.2-2020.02.14",
"_type": "_doc",
"_id": "AXiMQXABQLGiQgfNU2RI",
"_version": 1,
"_score": null,
"_source": {
"agent": {
"hostname": "xxx-www03",
"id": "1b299094-4d89-4e42-8fd7-0da4b29b6861",
"ephemeral_id": "0be31785-1825-4f33-96a2-63d423332046",
"type": "filebeat",
"version": "7.5.2"
},
"process": {
"pid": 32027
},
"log": {
"file": {
"path": "/var/log/httpd/error_log"
},
"offset": 1219779,
"level": "error"
},
"source": {
"geo": {
"continent_name": "Oceania",
"region_iso_code": "AU-NSW",
"city_name": "xxx",
"country_iso_code": "AU",
"region_name": "New South Wales",
"location": {
"lon": 151.1276,
"lat": -33.7811
}
},
"address": "10.2.2.2",
"port": "53625",
"ip": "10.2.2.2"
},
"message": "AH01102: error reading status line from remote server xxx:8080, referer: http://xxx/job/DataPath/",
"fileset": {
"name": "error"
},
"tags": [
"beats_input_codec_plain_applied",
"_jsonparsefailure"
],
"input": {
"type": "log"
},
"@timestamp": "2020-02-14T13:34:31.495+11:00",
"apache": {
"error": {
"module": "proxy_http"
}
},
"ecs": {
"version": "1.1.0"
},
"apr_error": {
"msg": "Internal error",
"code": 20014
},
"service": {
"type": "apache"
},
"host": {
"hostname": "xxx-www03",
"os": {
"kernel": "3.10.0-693.11.6.el7.x86_64",
"codename": "Core",
"name": "CentOS Linux",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": false,
"name": "xxx-www03",
"id": "d928eee246454138b9422fd86846ead5",
"architecture": "x86_64"
},
"@version": "1",
"event": {
"timezone": "+11:00",
"module": "apache",
"dataset": "apache.error"
}
},
"fields": {
"suricata.eve.timestamp": [
"2020-02-14T02:34:31.495Z"
],
"@timestamp": [
"2020-02-14T02:34:31.495Z"
]
},
"highlight": {
"fileset.name": [
"@kibana-highlighted-field@error@/kibana-highlighted-field@"
]
},
"sort": [
1581647671495
]
}
#this is data from access log
{
"_index": "filebeat-7.5.2-2020.02.14",
"_type": "_doc",
"_id": "P7DsQXABFXJEtiofyRaH",
"_version": 1,
"_score": null,
"_source": {
"agent": {
"hostname": "xxx-www02",
"id": "e20f614a-0e83-447a-9747-f9685a754821",
"type": "filebeat",
"ephemeral_id": "0c283572-a11e-4b1b-9466-c9802961899b",
"version": "7.5.2"
},
"log": {
"file": {
"path": "/var/log/httpd/access_log"
},
"offset": 27340342
},
"source": {
"geo": {
"region_iso_code": "AU-NSW",
"continent_name": "Oceania",
"city_name": "xxx",
"country_iso_code": "AU",
"region_name": "New South Wales",
"location": {
"lon": 151.1276,
"lat": -33.7811
}
},
"address": "10.3.3.3",
"ip": "10.3.3.3"
},
"fileset": {
"name": "access"
},
"url": {
"original": "/rest/mywork/latest/status/notification/count?_=1581653999991"
},
"tags": [
"beats_input_codec_plain_applied"
],
"input": {
"type": "log"
},
"apache": {
"access": {}
},
"@timestamp": "2020-02-14T04:20:00.000Z",
"ecs": {
"version": "1.1.0"
},
"service": {
"type": "apache"
},
"host": {
"hostname": "xxx-www02",
"os": {
"kernel": "3.10.0-957.21.3.el7.x86_64",
"codename": "Core",
"name": "CentOS Linux",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": false,
"name": "xxx-www02",
"id": "d928eee246454138b9422fd86846ead5",
"architecture": "x86_64"
},
"@version": "1",
"http": {
"request": {
"referrer": "https://xxx.com/pages/viewpage.action?spaceKey=printsys&title=Software+Development+Environment",
"method": "GET"
},
"load": {
"seconds": 0,
"microseconds": 12648
},
"response": {
"status_code": 200,
"body": {
"bytes": 41
}
},
"version": "1.1"
},
"event": {
"created": "2020-02-14T04:20:00.934Z",
"module": "apache",
"dataset": "apache.access"
},
"user": {
"name": "-"
},
"user_agent": {
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36",
"os": {
"name": "Windows",
"version": "10",
"full": "Windows 10"
},
"name": "Chrome",
"device": {
"name": "Other"
},
"version": "80.0.3987.100"
}
},
"fields": {
"event.created": [
"2020-02-14T04:20:00.934Z"
],
"suricata.eve.timestamp": [
"2020-02-14T04:20:00.000Z"
],
"@timestamp": [
"2020-02-14T04:20:00.000Z"
]
},
"sort": [
1581654000000
]
}
in both data, the source.geo.location is geo_point and this field is used in with Geohash in map visualize. Why the data from access log is showing but not data from error log?