Hello guys!
I hope you are ok.
I'm coming from ES 5.x .
Around a year ago I had an objective to setup testing ELK service which I achieved. Due to various reasons I stopped using it and left pretty much as it is until last week. I see 6.x brings a lot of nice features and it is amazing you guys did really awesome work! However, there is one thing which I can't figure out is those mapping types. I have countless times this doc https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html you have prepared for users, but sorry, it is still not clear to me what is wrong. I'm setting up everything from 0, but this time with big potential and just want to be prepared as much as I can for the future. I can pretend that everything is ok and keep using current mapping types , but because you removing them in the future I need to be prepared for this or ideally - if you could advice me on what I need to do ?
Basically, this is my template:
{
"nginx-access": {
"order": 0,
"index_patterns": [
"nginx-access-*"
],
"settings": {
"index": {
"refresh_interval": "5s"
}
},
"mappings": {
"nginx-access": {
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "text"
},
"beat": {
"properties": {
"hostname": {
"type": "text"
},
"name": {
"type": "text"
},
"version": {
"type": "text"
}
}
},
"browserdevice": {
"type": "text"
},
"browsermajor": {
"type": "text"
},
"browserminor": {
"type": "text"
},
"browsername": {
"type": "text"
},
"browseros": {
"type": "text"
},
"browseros_major": {
"type": "text"
},
"browseros_minor": {
"type": "text"
},
"browseros_name": {
"type": "text"
},
"browserpatch": {
"type": "text"
},
"count": {
"type": "long"
},
"geoip": {
"properties": {
"area_code": {
"type": "long"
},
"city_name": {
"type": "text"
},
"continent_code": {
"type": "text"
},
"country_code2": {
"type": "text"
},
"country_code3": {
"type": "text"
},
"country_name": {
"type": "text"
},
"dma_code": {
"type": "long"
},
"ip": {
"type": "text"
},
"latitude": {
"type": "double"
},
"location": {
"type": "geo_point"
},
"longitude": {
"type": "double"
},
"postal_code": {
"type": "text"
},
"real_region_name": {
"type": "text"
},
"region_name": {
"type": "text"
},
"timezone": {
"type": "text"
}
}
},
"input_type": {
"type": "text"
},
"offset": {
"type": "long"
},
"our_server": {
"type": "text"
},
"prospector": {
"properties": {
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"referer": {
"type": "text"
},
"request": {
"type": "text"
},
"request_arrival_time": {
"type": "text"
},
"request_body_bytes_sent": {
"type": "text"
},
"request_ip": {
"type": "text"
},
"request_length": {
"type": "text"
},
"request_processing_time": {
"type": "text"
},
"request_response_code": {
"type": "text"
},
"source": {
"type": "text"
},
"tags": {
"type": "text"
},
"type": {
"type": "text"
},
"upstream_response_time": {
"type": "text"
},
"user_agent": {
"type": "text"
},
"connection_serial_number": {
"type": "long"
}
}
}
},
"aliases": {}
}
}
And these are my input/filter/output stuff:
root@ELK:/etc/logstash/conf.d# cat input.conf
input {
beats {
port => 5044
ssl => true
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
root@ELK:/etc/logstash/conf.d# cat filter-nginx_access.conf
filter {
mutate {
remove_tag => ["beats_input_codec_plain_applied"]
}
if "nginx-access" in [tags] {
grok {
patterns_dir => ["/etc/logstash/patterns"]
match => {"message" => "%{NGINXACCESS}"}
}
useragent {
source => "user_agent"
prefix => "browser"
}
geoip {
source => "request_ip"
}
}
mutate {
remove_field => ["browserbuild", "message", "fields", "beat.hostname", "beat.name", "host"]
gsub => ["request_arrival_time", "(.{6}$)", ""]
}
}
root@ELK:/etc/logstash/conf.d# cat output.conf
output {
if "nginx-access" in [tags] {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "nginx-access"
}
}
}
Everything works, but I get messages about mapping types being deprecated. I just want to have a single index for now where one of the fields would be with geo_point, but also, I want to understand which strategies I would need to use, if let say I will want to have different index with different mappings ? With configuration which would be ready to use in 7.x onwards.
Thanks!