- Our ES version is old, but we're working to upgrade. I know this adds a big variable.
- No data tiering or movement to warm/cold. Data is deleted from cluster after ~6 months.
- Non-current indices are force-merged.
- One issue we're dealing with that may be related: We were creating fresh snapshot repos (S3 backed) at the beginning of every months, and by EOM we'd have >800 snapshots in that repo, significantly degrading snapshot performance. That has been fixed for Oct - one long-lived repo, and logic to delete snaps and keep count <50.
Here is output from _cluster/stats:
{
"_nodes" : {
"total" : 36,
"successful" : 36,
"failed" : 0
},
"cluster_name" : "cluster",
"timestamp" : 1664880527851,
"status" : "green",
"indices" : {
"count" : 112,
"shards" : {
"total" : 6921,
"primaries" : 2299,
"replication" : 2.0104393214441063,
"index" : {
"shards" : {
"min" : 2,
"max" : 72,
"avg" : 61.794642857142854
},
"primaries" : {
"min" : 1,
"max" : 24,
"avg" : 20.526785714285715
},
"replication" : {
"min" : 1.0,
"max" : 32.0,
"avg" : 2.25
}
}
},
"docs" : {
"count" : 20007624527,
"deleted" : 46734007
},
"store" : {
"size" : "101.1tb",
"size_in_bytes" : 111224919772367,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"fielddata" : {
"memory_size" : "12.2gb",
"memory_size_in_bytes" : 13183287448,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "15.3gb",
"memory_size_in_bytes" : 16514077608,
"total_count" : 58118164221,
"hit_count" : 27773552851,
"miss_count" : 30344611370,
"cache_size" : 552739,
"cache_count" : 48375813,
"evictions" : 47823074
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 84559,
"memory" : "164.6gb",
"memory_in_bytes" : 176749193869,
"terms_memory" : "127.3gb",
"terms_memory_in_bytes" : 136765097675,
"stored_fields_memory" : "23.8gb",
"stored_fields_memory_in_bytes" : 25562813472,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "5.2mb",
"norms_memory_in_bytes" : 5453376,
"points_memory" : "1.7gb",
"points_memory_in_bytes" : 1839953534,
"doc_values_memory" : "11.7gb",
"doc_values_memory_in_bytes" : 12575875812,
"index_writer_memory" : "790.6mb",
"index_writer_memory_in_bytes" : 829097505,
"version_map_memory" : "3.1mb",
"version_map_memory_in_bytes" : 3323883,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : -1,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 36,
"data" : 33,
"coordinating_only" : 0,
"master" : 3,
"ingest" : 36
},
"versions" : [
"5.3.3"
],
"os" : {
"available_processors" : 462,
"allocated_processors" : 462,
"names" : [
{
"name" : "Linux",
"count" : 36
}
],
"mem" : {
"total" : "3.4tb",
"total_in_bytes" : 3828682395648,
"free" : "104.1gb",
"free_in_bytes" : 111856631808,
"used" : "3.3tb",
"used_in_bytes" : 3716825763840,
"free_percent" : 3,
"used_percent" : 97
}
},
"process" : {
"cpu" : {
"percent" : 75
},
"open_file_descriptors" : {
"min" : 1095,
"max" : 1880,
"avg" : 1672
}
},
"jvm" : {
"max_uptime" : "290.4d",
"max_uptime_in_millis" : 25092630336,
"versions" : [
{
"version" : "1.8.0_312",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.312-b07",
"vm_vendor" : "Amazon.com Inc.",
"count" : 1
},
{
"version" : "1.8.0_222",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.222-b10",
"vm_vendor" : "Amazon.com Inc.",
"count" : 35
}
],
"mem" : {
"heap_used" : "583.6gb",
"heap_used_in_bytes" : 626680627560,
"heap_max" : "998.1gb",
"heap_max_in_bytes" : 1071752282112
},
"threads" : 6413
},
"fs" : {
"total" : "339.8tb",
"total_in_bytes" : 373654192852992,
"free" : "238.1tb",
"free_in_bytes" : 261850661273600,
"available" : "238.1tb",
"available_in_bytes" : 261850661273600
},
"plugins" : [
{
"name" : "search-guard",
"version" : "5.3.3-1",
"description" : "Provide access control related features for Elasticsearch 5",
"classname" : "com.floragunn.searchguard.SearchGuardPlugin"
},
{
"name" : "discovery-ec2",
"version" : "5.3.3",
"description" : "The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.",
"classname" : "org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin"
},
{
"name" : "repository-s3",
"version" : "5.3.3",
"description" : "The S3 repository plugin adds S3 repositories",
"classname" : "org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin"
}
],
"network_types" : {
"transport_types" : {
"com.floragunn.searchguard.ssl.http.netty.SearchGuardSSLNettyTransport" : 36
},
"http_types" : {
"com.floragunn.searchguard.http.SearchGuardHttpServerTransport" : 36
}
}
}
}