Didn't succeed to migrate my data from the elastic engineer I course to elasticsearch 7.0.
Input log:
{"geoip":{"city_name":"Holly Springs","country_name":"United States","country_code2":"US","continent_code":"NA","country_code3":"US","location":{"lon":-78.8786,"lat":35.6022},"region_name":"North Carolina"},"status_code":200,"method":"GET","level":"info","runtime_ms":97,"http_version":"1.1","language":{"url":"/blog/elasticsearch-5-3-0-released?utm_source=dbweekly&utm_medium=email","code":"en-us"},"originalUrl":"/blog/elasticsearch-5-3-0-released?utm_source=dbweekly&utm_medium=email","response_size":49766,"@timestamp":"2017-03-31T23:47:27.564Z","host":"server1","user_agent":"Amazon CloudFront"}
filebeat.yml:
filebeat.inputs:
- type: log
enabled: true
paths:- /path/data/elastic_blog_curated_access_logs_server*/*.log
json.keys_under_root: true
json.overwrite_keys: true
- /path/data/elastic_blog_curated_access_logs_server*/*.log
processors:
- drop_fields:
fields: ["host.name"]
-
decode_json_fields:
fields: ["json"]
process_array: true
target: ''
overwrite_keys: true -
drop_fields:
fields: ["log", "input", "ecs", "agent"]
setup.template.enabled: true
setup.template.name: "logs-"
setup.template.pattern: "logs-*"
setup.dashboards.enabled: false
#output.console.pretty: true
output.elasticsearch:
hosts: ["localhost:9200"]
username: elastic
password: password
index: "logs_%{[host]}"
bulk_max_size: 100
Result:
{
"@timestamp": "2017-03-31T23:56:29.072Z",
"@metadata": {
"beat": "",
"type": "_doc",
"version": ""
},
"geoip": {
"continent_code": "NA",
"country_code3": "US",
"location": {
"lon": -77.4728,
"lat": 39.0481
},
"region_name": "Virginia",
"city_name": "Ashburn",
"country_code2": "US",
"country_name": "United States"
},
"level": "info",
"host": {
"name": "mark-busenharts-imac-4.home"
},
"status_code": 200,
"runtime_ms": 146,
"language": {
"url": "/blog/author/thom-o%27connor",
"code": "en-us"
},
"response_size": 45120,
"method": "GET",
"http_version": "1.1",
"user_agent": "Amazon CloudFront",
"originalUrl": "/blog/author/thom-o%27connor"
}
which is pretty much what I want except for the host field that should be "server1". In my understanding the host value should be overwritten with overwrite_keys: true. What am I missing.
Would be great if somebody could give me a hint on how to get the correct value in the host field.
Kind regards
Mark