Hello,
I try to add a field with vhost name. Actually, the filebeat module apache2 work but it dosen't show the vhost name.
My apache log looks like this:
www.foo.com:443 37.34.55.121 - - [11/Aug/2017:12:07:53 +0200] "GET /sites/default/files/pdf/tauxphnl.pdf HTTP/1.1" 304 5163 "-" "Mozilla/5.0 (compatible; MSIE 9.11; Windows NT 6.1; Trident/5.0)"
and the result in kibana:
{
"_index": "filebeat-2017.08.11",
"_type": "doc",
"_id": "AV3QvyqOs-yNO51h3JDi",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2017-08-11T10:02:29.000Z",
"offset": 4641569,
"apache2": {
"access": {
"referrer": "https://xxx.xxxxxxxx.xxx/xx/xxxxxxxxxxxxx",
"response_code": "304",
"remote_ip": "XXX.XXX.XXX.XXX",
"geoip": {
"continent_name": "Europe",
"country_iso_code": "BE",
"location": {
"lon": 4.35,
"lat": 50.85
}
},
"method": "GET",
"user_name": "-",
"http_version": "1.1",
"body_sent": {
"bytes": "5162"
},
"url": "/sites/default/files/xxxxx-xxxxx-xxx.png",
"user_agent": {
"patch": "2171",
"major": "39",
"minor": "0",
"os": "Windows 8.1",
"name": "Chrome",
"os_name": "Windows 8.1",
"device": "Other"
}
}
},
"beat": {
"hostname": "xxxxx01",
"name": "xxxxx01",
"version": "5.5.1"
},
"input_type": "log",
"read_timestamp": "2017-08-11T10:02:35.602Z",
"source": "/var/log/apache2/access.log",
"type": "log"
},
"fields": {
"@timestamp": [
1502445749000
]
I need to see in kibana the vhost name (first field in log) where 's the info before the remote_ip?
I try to update the /usr/share/filebeat/modules/apache2/access/ingest/defult.json like this:
{
"description": "Pipeline for parsing Apache2 access logs. Requires the geoip and user_agent plugins.",
"processors": [{
"grok": {
"field": "message",
"patterns":[
"%{HOSTPORT:vhost} %{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \[%{HTTPDATE:apache2.access.time}\] "%{WORD:apache2.access.method} %{DATA:apache2.access.url} HTTP/%{NUMBER:apache2.access.http_version}" %{NUMBER:apache2.access.response_code} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( "%{DATA:apache2.access.referrer}")?( "%{DATA:apache2.access.agent}")?",
"%{HOSTPORT:vhost} %{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \[%{HTTPDATE:apache2.access.time}\] "-" %{NUMBER:apache2.access.response_code} -"
],
"ignore_missing": true
}
},{
"remove":{
"field": "message"
}
}, {
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
}, {
"date": {
"field": "apache2.access.time",
"target_field": "@timestamp",
"formats": ["dd/MMM/YYYY:H:m:s Z"]
}
}, {
"remove": {
"field": "apache2.access.time"
}
}, {
"user_agent": {
"field": "apache2.access.agent",
"target_field": "apache2.access.user_agent",
"ignore_failure": true
}
}, {
"remove": {
"field": "apache2.access.agent",
"ignore_failure": true
}
}, {
"geoip": {
"field": "apache2.access.remote_ip",
"target_field": "apache2.access.geoip"
}
}],
"on_failure" : [{
"set" : {
"field" : "error",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}
I have added the field: %{HOSTPORT:vhost} %{IPORHOST:apache2.access.remote_ip} to match the vhostname and in /etc/filebeat/filebeat.template.json I add this:
{
"mappings": {
"default": {
"_all": {
"norms": false
},
"_meta": {
"version": "5.5.1"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"vhost": {
"type": "keyword"
},
"apache2": {
"properties": {
"access": {
"properties": {
"message": {
"type": "text"
},
"vhost_name": {
"type": "text"
},
"agent": {
"norms": false,
"type": "text"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"geoip": {
"properties": {
"continent_name": {
"ignore_above": 1024,
"type": "keyword"
},
"country_iso_code": {
"ignore_above": 1024,
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},.......
How do this? Anyone can help me please?