Filebeat + Apache2 Module + add field


(Robin ) #1

Hello,

I try to add a field with vhost name. Actually, the filebeat module apache2 work but it dosen't show the vhost name.

My apache log looks like this:
www.foo.com:443 37.34.55.121 - - [11/Aug/2017:12:07:53 +0200] "GET /sites/default/files/pdf/tauxphnl.pdf HTTP/1.1" 304 5163 "-" "Mozilla/5.0 (compatible; MSIE 9.11; Windows NT 6.1; Trident/5.0)"

and the result in kibana:

{
"_index": "filebeat-2017.08.11",
"_type": "doc",
"_id": "AV3QvyqOs-yNO51h3JDi",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2017-08-11T10:02:29.000Z",
"offset": 4641569,
"apache2": {
"access": {
"referrer": "https://xxx.xxxxxxxx.xxx/xx/xxxxxxxxxxxxx",
"response_code": "304",
"remote_ip": "XXX.XXX.XXX.XXX",
"geoip": {
"continent_name": "Europe",
"country_iso_code": "BE",
"location": {
"lon": 4.35,
"lat": 50.85
}
},
"method": "GET",
"user_name": "-",
"http_version": "1.1",
"body_sent": {
"bytes": "5162"
},
"url": "/sites/default/files/xxxxx-xxxxx-xxx.png",
"user_agent": {
"patch": "2171",
"major": "39",
"minor": "0",
"os": "Windows 8.1",
"name": "Chrome",
"os_name": "Windows 8.1",
"device": "Other"
}
}
},
"beat": {
"hostname": "xxxxx01",
"name": "xxxxx01",
"version": "5.5.1"
},
"input_type": "log",
"read_timestamp": "2017-08-11T10:02:35.602Z",
"source": "/var/log/apache2/access.log",
"type": "log"
},
"fields": {
"@timestamp": [
1502445749000
]

I need to see in kibana the vhost name (first field in log) where 's the info before the remote_ip?

I try to update the /usr/share/filebeat/modules/apache2/access/ingest/defult.json like this:

{
"description": "Pipeline for parsing Apache2 access logs. Requires the geoip and user_agent plugins.",
"processors": [{
"grok": {
"field": "message",
"patterns":[
"%{HOSTPORT:vhost} %{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \[%{HTTPDATE:apache2.access.time}\] "%{WORD:apache2.access.method} %{DATA:apache2.access.url} HTTP/%{NUMBER:apache2.access.http_version}" %{NUMBER:apache2.access.response_code} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( "%{DATA:apache2.access.referrer}")?( "%{DATA:apache2.access.agent}")?",
"%{HOSTPORT:vhost} %{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \[%{HTTPDATE:apache2.access.time}\] "-" %{NUMBER:apache2.access.response_code} -"
],
"ignore_missing": true
}
},{
"remove":{
"field": "message"
}
}, {
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
}, {
"date": {
"field": "apache2.access.time",
"target_field": "@timestamp",
"formats": ["dd/MMM/YYYY:H:m:s Z"]
}
}, {
"remove": {
"field": "apache2.access.time"
}
}, {
"user_agent": {
"field": "apache2.access.agent",
"target_field": "apache2.access.user_agent",
"ignore_failure": true
}
}, {
"remove": {
"field": "apache2.access.agent",
"ignore_failure": true
}
}, {
"geoip": {
"field": "apache2.access.remote_ip",
"target_field": "apache2.access.geoip"
}
}],
"on_failure" : [{
"set" : {
"field" : "error",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}

I have added the field: %{HOSTPORT:vhost} %{IPORHOST:apache2.access.remote_ip} to match the vhostname and in /etc/filebeat/filebeat.template.json I add this:

{
"mappings": {
"default": {
"_all": {
"norms": false
},
"_meta": {
"version": "5.5.1"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"vhost": {
"type": "keyword"
},
"apache2": {
"properties": {
"access": {
"properties": {
"message": {
"type": "text"
},
"vhost_name": {
"type": "text"
},
"agent": {
"norms": false,
"type": "text"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"geoip": {
"properties": {
"continent_name": {
"ignore_above": 1024,
"type": "keyword"
},
"country_iso_code": {
"ignore_above": 1024,
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},.......

How do this? Anyone can help me please?


(Andrew Kroh) #2

After modifying the pipeline JSON file on the Filebeat host you need to delete the pipeline from Elasticsearch. If you don't delete the pipeline from Elasticsearch, then Filebeat will not update the pipeline config with the new settings.

Additionally when changing the filebeat index template file you need to delete the template from Elasticsearch too. Or you can tell Filebeat to overwrite the template.


(system) #3

This topic was automatically closed after 21 days. New replies are no longer allowed.