Thank you @DavidTurner for the quick reply. I just discovered that command as you were responding and came back to post my findings.
It turns out that many of my shards are culprits. Looking something like this:
"env-stg-2020.02.20" : {
"uuid" : "TgkrNGtKTwGNDeSshJ2YLQ",
"primaries" : {
"fielddata" : {
"memory_size_in_bytes" : 64716808,
"evictions" : 0,
"fields" : {
"logmessage.keyword" : {
"memory_size_in_bytes" : 225728
},
"fields.service.keyword" : {
"memory_size_in_bytes" : 2096
},
"fields.env.keyword" : {
"memory_size_in_bytes" : 1088
},
"_id" : {
"memory_size_in_bytes" : 64487896
}
}
}
},
My template for this index looks like this:
"default" : {
"order" : -1,
"index_patterns" : [
"env-stg*",
"env-prd*",
"env2-stg*",
"env2-prd*"
],
"settings" : {
"index" : {
"number_of_shards" : "2",
"number_of_replicas" : "0"
}
},
"mappings" : {
"doc" : {
"properties" : {
"logTimestamphttp" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"auth" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"ident" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"source" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"lc_identifier" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"logTimestamp" : {
"type" : "date"
},
"clientip" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"beat" : {
"properties" : {
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
}
}
},
"host" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"logTimestampString" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"class" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"offset" : {
"type" : "long"
},
"logmessage" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"verb" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"prospector" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
}
}
},
"thread" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"input" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
}
}
},
"prelogmessage" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"@timestamp" : {
"type" : "date"
},
"bytes" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"response" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"loglevel" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"httpversion" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"fields" : {
"properties" : {
"product" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"service" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"env" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
},
"customer" : {
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
}
}
}
}
}
}
},
"aliases" : { }
},
It looks like I overlooked the template in testing when comparing for deltas between the production and testing environments. However, it appears everything is working how I'd expect in testing. I believe the template currently in use in production was cherry-picked and persisted since we were on version 5. I don't believe we were using it to do anything specific other than set the number of shards/replicas originally.
As a reference, the testing environment looks like this:
{
"_shards" : {
"total" : 239,
"successful" : 239,
"failed" : 0
},
"_all" : {
"primaries" : {
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"fields" : {
"type" : {
"memory_size_in_bytes" : 0
},
"_id" : {
"memory_size_in_bytes" : 0
}
}
}
},
I imagine removing this template altogether is an option. Or, replacing with the template that exists in our testing environment currently. Is there a better option?