I have a 3 node cluster. Once a day I create a new index and bulk insert some data. Then throughout the day singular docs are slowly added to that same index. Once a new index has been created and fulfilled the previous days index is deleted. Once a month I update this cluster and reboot the servers it is on. During the month the index time starts at a few seconds and then slowly creeps up to several hours in a fairly linear fashion. Only restarting Elasticsearch resets it. What could this be?
Welcome to our community! 
What is the output from the _cluster/stats?pretty&human API?
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "main-production-app",
"cluster_uuid" : "OgZjEJvjT26hz6eIWlS9CQ",
"timestamp" : 1644224921926,
"status" : "green",
"indices" : {
"count" : 83,
"shards" : {
"total" : 194,
"primaries" : 91,
"replication" : 1.1318681318681318,
"index" : {
"shards" : {
"min" : 2,
"max" : 9,
"avg" : 2.3373493975903616
},
"primaries" : {
"min" : 1,
"max" : 3,
"avg" : 1.0963855421686748
},
"replication" : {
"min" : 1.0,
"max" : 2.0,
"avg" : 1.0481927710843373
}
}
},
"docs" : {
"count" : 3192571,
"deleted" : 1545042
},
"store" : {
"size" : "7.7gb",
"size_in_bytes" : 8280619895,
"total_data_set_size" : "7.7gb",
"total_data_set_size_in_bytes" : 8280619895,
"reserved" : "0b",
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size" : "0b",
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "100.1kb",
"memory_size_in_bytes" : 102552,
"total_count" : 12108777,
"hit_count" : 412441,
"miss_count" : 11696336,
"cache_size" : 8,
"cache_count" : 16831,
"evictions" : 16823
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 757,
"memory" : "6.4mb",
"memory_in_bytes" : 6730934,
"terms_memory" : "3.9mb",
"terms_memory_in_bytes" : 4172064,
"stored_fields_memory" : "364.3kb",
"stored_fields_memory_in_bytes" : 373096,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "365.8kb",
"norms_memory_in_bytes" : 374592,
"points_memory" : "0b",
"points_memory_in_bytes" : 0,
"doc_values_memory" : "1.7mb",
"doc_values_memory_in_bytes" : 1811182,
"index_writer_memory" : "100.6mb",
"index_writer_memory_in_bytes" : 105584860,
"version_map_memory" : "7.5mb",
"version_map_memory_in_bytes" : 7880372,
"fixed_bit_set" : "866kb",
"fixed_bit_set_memory_in_bytes" : 886864,
"max_unsafe_auto_id_timestamp" : 1644192001550,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "boolean",
"count" : 89,
"index_count" : 23,
"script_count" : 0
},
{
"name" : "constant_keyword",
"count" : 9,
"index_count" : 3,
"script_count" : 0
},
{
"name" : "date",
"count" : 206,
"index_count" : 56,
"script_count" : 0
},
{
"name" : "double",
"count" : 4,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "float",
"count" : 44,
"index_count" : 6,
"script_count" : 0
},
{
"name" : "half_float",
"count" : 47,
"index_count" : 11,
"script_count" : 0
},
{
"name" : "integer",
"count" : 170,
"index_count" : 11,
"script_count" : 0
},
{
"name" : "ip",
"count" : 5,
"index_count" : 5,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 998,
"index_count" : 56,
"script_count" : 0
},
{
"name" : "long",
"count" : 1267,
"index_count" : 49,
"script_count" : 0
},
{
"name" : "nested",
"count" : 51,
"index_count" : 37,
"script_count" : 0
},
{
"name" : "object",
"count" : 1014,
"index_count" : 56,
"script_count" : 0
},
{
"name" : "text",
"count" : 222,
"index_count" : 52,
"script_count" : 0
},
{
"name" : "version",
"count" : 2,
"index_count" : 2,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [
{
"name" : "ngram",
"count" : 4,
"index_count" : 4
}
],
"filter_types" : [ ],
"analyzer_types" : [
{
"name" : "custom",
"count" : 4,
"index_count" : 4
}
],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [
{
"name" : "lowercase",
"count" : 4,
"index_count" : 4
}
],
"built_in_analyzers" : [
{
"name" : "simple",
"count" : 14,
"index_count" : 2
}
]
},
"versions" : [
{
"version" : "7.2.0",
"index_count" : 2,
"primary_shard_count" : 2,
"total_primary_size" : "214.5kb",
"total_primary_bytes" : 219669
},
{
"version" : "7.3.0",
"index_count" : 1,
"primary_shard_count" : 1,
"total_primary_size" : "45.5kb",
"total_primary_bytes" : 46642
},
{
"version" : "7.4.0",
"index_count" : 3,
"primary_shard_count" : 3,
"total_primary_size" : "58.7kb",
"total_primary_bytes" : 60159
},
{
"version" : "7.5.0",
"index_count" : 2,
"primary_shard_count" : 2,
"total_primary_size" : "56.7kb",
"total_primary_bytes" : 58137
},
{
"version" : "7.6.1",
"index_count" : 2,
"primary_shard_count" : 2,
"total_primary_size" : "87.3kb",
"total_primary_bytes" : 89421
},
{
"version" : "7.7.1",
"index_count" : 2,
"primary_shard_count" : 2,
"total_primary_size" : "145.7kb",
"total_primary_bytes" : 149246
},
{
"version" : "7.8.0",
"index_count" : 3,
"primary_shard_count" : 3,
"total_primary_size" : "4.4mb",
"total_primary_bytes" : 4689544
},
{
"version" : "7.9.1",
"index_count" : 2,
"primary_shard_count" : 2,
"total_primary_size" : "20.9mb",
"total_primary_bytes" : 21924856
},
{
"version" : "7.10.1",
"index_count" : 3,
"primary_shard_count" : 3,
"total_primary_size" : "11.7mb",
"total_primary_bytes" : 12277540
},
{
"version" : "7.11.0",
"index_count" : 2,
"primary_shard_count" : 2,
"total_primary_size" : "10.4mb",
"total_primary_bytes" : 10936444
},
{
"version" : "7.15.0",
"index_count" : 9,
"primary_shard_count" : 9,
"total_primary_size" : "73.1kb",
"total_primary_bytes" : 74897
},
{
"version" : "7.15.1",
"index_count" : 9,
"primary_shard_count" : 9,
"total_primary_size" : "89.7kb",
"total_primary_bytes" : 91950
},
{
"version" : "7.15.2",
"index_count" : 10,
"primary_shard_count" : 10,
"total_primary_size" : "166.3kb",
"total_primary_bytes" : 170378
},
{
"version" : "7.16.2",
"index_count" : 18,
"primary_shard_count" : 18,
"total_primary_size" : "818.2mb",
"total_primary_bytes" : 858001297
},
{
"version" : "7.17.0",
"index_count" : 15,
"primary_shard_count" : 23,
"total_primary_size" : "2.2gb",
"total_primary_bytes" : 2400477152
}
]
},
"nodes" : {
"count" : {
"total" : 3,
"coordinating_only" : 0,
"data" : 3,
"data_cold" : 3,
"data_content" : 3,
"data_frozen" : 3,
"data_hot" : 3,
"data_warm" : 3,
"ingest" : 3,
"master" : 3,
"ml" : 3,
"remote_cluster_client" : 3,
"transform" : 3,
"voting_only" : 0
},
"versions" : [
"7.17.0"
],
"os" : {
"available_processors" : 3,
"allocated_processors" : 3,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 3
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 3
}
],
"mem" : {
"total" : "10.3gb",
"total_in_bytes" : 11128025088,
"free" : "475.2mb",
"free_in_bytes" : 498307072,
"used" : "9.8gb",
"used_in_bytes" : 10629718016,
"free_percent" : 4,
"used_percent" : 96
}
},
"process" : {
"cpu" : {
"percent" : 12
},
"open_file_descriptors" : {
"min" : 680,
"max" : 734,
"avg" : 699
}
},
"jvm" : {
"max_uptime" : "2.9d",
"max_uptime_in_millis" : 257377632,
"versions" : [
{
"version" : "17.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "17.0.1+12",
"vm_vendor" : "Eclipse Adoptium",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 3
}
],
"mem" : {
"heap_used" : "1.3gb",
"heap_used_in_bytes" : 1458764280,
"heap_max" : "3gb",
"heap_max_in_bytes" : 3221225472
},
"threads" : 207
},
"fs" : {
"total" : "149.9gb",
"total_in_bytes" : 161024593920,
"free" : "116.7gb",
"free_in_bytes" : 125377818624,
"available" : "116.7gb",
"available_in_bytes" : 125377818624
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 3
},
"http_types" : {
"security4" : 3
}
},
"discovery_types" : {
"zen" : 3
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "rpm",
"count" : 3
}
],
"ingest" : {
"number_of_pipelines" : 2,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time" : "0s",
"time_in_millis" : 0
}
}
}
}
}
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.