I am using the filebeat to transfer nginx access log to elasticsearch with json format .
access.log
{ "@timestamp": "05/Jan/2021:13:38:28 +0800", "pid": "52336", "connection": "217062444", "connection_requests": "18", "http_x_forwarded_for": "-", "upstream_addr": "10.7.4.146:8082", "upstream_connect_time": "0.036", "upstream_response_length": "242","upstream_response_time": "0.077","remote_addr": "10.8.4.253","remote_port": "52918","request": "POST /njdf_ajax?returnType=json&service=P1005000&encoderType=none HTTP/1.1", "request_length": "1117", "content_length": "492", "request_time": "0.077", "status": "200", "bytes_sent": "422", "body_bytes_sent": "253","http_user_agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" }
The filebeat.yml as below:
filebeat.inputs:
- type: log
paths:
# - /usr/local/CCS-20201207161336.txt
# - /usr/local/testlog4.log
# - /usr/local/access.log
- /root/json/json.log
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_keys: true
#output.logstash:
# hosts: ["0.0.0.0:5044"]
setup.kibana:
hosts: "10.239.113.77:5601"
setup.ilm.enabled: false
setup.template.overwrite: true
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
output.elasticsearch:
hosts: ["10.239.113.77:9200"]
index: "nginx-sz-%{+yyyy-MM}"
processors:
- add_locale: ~
But the elasticsearch can't parse @timestamp correctly , does anyone have similar issue ?
I tried to use custom data format in template mappings , but still can't fix it .