Hello,
recently I've upgraded my ELK from 5.6 to 6.2(and logstash's mapping templates too). And now I've noticed that new indices almost twice bigger than old indices with same amount of documents inside.
Here's logstash mapping:
{
"template" : "logs-mail.log-*",
"settings" : {
"index.refresh_interval" : "5s",
"number_of_shards": 2,
"number_of_replicas": 0
},
"mappings" : {
"_default_" : {
"_all" : { "enabled" : false },
"dynamic_templates" : [{
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "keyword", "omit_norms" : true,
"fielddata" : { "format" : "disabled" },
"fields" : {
"raw" : {"type": "keyword", "ignore_above" : 256}
"properties" : {
"@timestamp" : { "type" : "date" },
"index_date" : { "type" : "keyword" },
"server" : { "type" : "keyword" },
"program" : { "type" : "keyword" },
"dsn" : { "type" : "keyword" },
"delay" : { "type" : "float" },
"delays" : { "type" : "keyword" },
"relay-ip" : { "type" : "ip" },
"send-to" : { "type" : "keyword" },
"delay" : { "type" : "float" },
"conn_use" : { "type" : "integer" },
"relay-port" : { "type" : "integer" },
"queue-id" : { "type" : "keyword" },
"status" : { "type" : "keyword" },
"domain-rcpt" : { "type" : "keyword" },
"relay-domain" : { "type" : "keyword" }
And mapping for new created index:
{
"logs-mail.log-2018.03.30": {
"mappings": {
"doc": {
"_all": {
"enabled": false
},
"dynamic_templates": [
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"ignore_above": 256,
"type": "keyword"
}
},
"omit_norms": true,
"type": "keyword"
}
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"conn_use": {
"type": "integer"
},
"delay": {
"type": "float"
},
"delays": {
"type": "keyword"
},
"domain-rcpt": {
"type": "keyword"
},
"dsn": {
"type": "keyword"
},
"index_date": {
"type": "keyword"
},
"message": {
"type": "keyword",
"fields": {
"raw": {
"type": "keyword",
"ignore_above": 256
}
}
},
"program": {
"type": "keyword"
},
"queue-id": {
"type": "keyword"
},
"relay": {
"type": "keyword",
"fields": {
"raw": {
"type": "keyword",
"ignore_above": 256
}
}
},
"relay-domain": {
"type": "keyword"
},
"relay-ip": {
"type": "ip"
},
"relay-port": {
"type": "integer"
},
"send-to": {
"type": "keyword"
},
"server": {
"type": "keyword"
},
"status": {
"type": "keyword"
},
"tags": {
"type": "keyword",
"fields": {
"raw": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "keyword",
"fields": {
"raw": {
"type": "keyword",
"ignore_above": 256
"_default_": {
"_all": {
"enabled": false
},
"dynamic_templates": [
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"fielddata": {
"format": "disabled"
},
"fields": {
"raw": {
"ignore_above": 256,
"type": "keyword"
}
},
"omit_norms": true,
"type": "keyword"
"properties": {
"@timestamp": {
"type": "date"
},
"conn_use": {
"type": "integer"
},
"delay": {
"type": "float"
},
"delays": {
"type": "keyword"
},
"domain-rcpt": {
"type": "keyword"
},
"dsn": {
"type": "keyword"
},
"index_date": {
"type": "keyword"
},
"program": {
"type": "keyword"
},
"queue-id": {
"type": "keyword"
},
"relay-domain": {
"type": "keyword"
},
"relay-ip": {
"type": "ip"
},
"relay-port": {
"type": "integer"
},
"send-to": {
"type": "keyword"
},
"server": {
"type": "keyword"
},
"status": {
"type": "keyword"
}
*I've deleted braces to circumvent 7000 char limit
Looks like new indices is creating with extra fields, but why?