I'm using elasticsearch as database but I'm facing momentary surge in search rate and latency. Can anyone tell why there is a high search rate and latency observerd I have 10 nodes of cluster which are physical vms. It's working fine as of now but and I'm putting the below stats of cluster:
{
"_nodes" : {
"total" : 10,
"successful" : 10,
"failed" : 0
},
"cluster_name" : "graphreadz2vs",
"cluster_uuid" : "D63jtD0hahgeuk3js8ja9dw2mEdGSw",
"timestamp" : 1714392526175,
"status" : "green",
"indices" : {
"count" : 39,
"shards" : {
"total" : 197,
"primaries" : 62,
"replication" : 2.1774193548387095,
"index" : {
"shards" : {
"min" : 2,
"max" : 72,
"avg" : 5.051282051282051
},
"primaries" : {
"min" : 1,
"max" : 9,
"avg" : 1.5897435897435896
},
"replication" : {
"min" : 1.0,
"max" : 7.0,
"avg" : 1.2564102564102564
}
}
},
"docs" : {
"count" : 619585818,
"deleted" : 118392189
},
"store" : {
"size_in_bytes" : 838169390789,
"total_data_set_size_in_bytes" : 838169390789,
"reserved_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 14112,
"evictions" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 5441447555,
"total_count" : 3109663468713,
"hit_count" : 122009856826,
"miss_count" : 2987653611887,
"cache_size" : 986242,
"cache_count" : 5177072,
"evictions" : 4190830
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 2780,
"memory_in_bytes" : 42113584,
"terms_memory_in_bytes" : 22388376,
"stored_fields_memory_in_bytes" : 6534432,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 367296,
"points_memory_in_bytes" : 0,
"doc_values_memory_in_bytes" : 12823480,
"index_writer_memory_in_bytes" : 3372616,
"version_map_memory_in_bytes" : 3031,
"fixed_bit_set_memory_in_bytes" : 655860408,
"max_unsafe_auto_id_timestamp" : 1714371858260,
"file_sizes" : { }
},
"mappings" : {
"field_types" : [
{
"name" : "boolean",
"count" : 55,
"index_count" : 21,
"script_count" : 0
},
{
"name" : "constant_keyword",
"count" : 6,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "date",
"count" : 128,
"index_count" : 30,
"script_count" : 0
},
{
"name" : "float",
"count" : 83,
"index_count" : 13,
"script_count" : 0
},
{
"name" : "half_float",
"count" : 56,
"index_count" : 14,
"script_count" : 0
},
{
"name" : "integer",
"count" : 179,
"index_count" : 11,
"script_count" : 0
},
{
"name" : "ip",
"count" : 2,
"index_count" : 2,
"script_count" : 0
},
{
"name" : "keyword",
"count" : 807,
"index_count" : 30,
"script_count" : 0
},
{
"name" : "long",
"count" : 1325,
"index_count" : 25,
"script_count" : 0
},
{
"name" : "nested",
"count" : 33,
"index_count" : 13,
"script_count" : 0
},
{
"name" : "object",
"count" : 885,
"index_count" : 25,
"script_count" : 0
},
{
"name" : "text",
"count" : 103,
"index_count" : 20,
"script_count" : 0
},
{
"name" : "version",
"count" : 4,
"index_count" : 4,
"script_count" : 0
}
],
"runtime_field_types" : [ ]
},
"analysis" : {
"char_filter_types" : [ ],
"tokenizer_types" : [ ],
"filter_types" : [ ],
"analyzer_types" : [ ],
"built_in_char_filters" : [ ],
"built_in_tokenizers" : [ ],
"built_in_filters" : [ ],
"built_in_analyzers" : [ ]
},
"versions" : [
{
"version" : "7.8.1",
"index_count" : 6,
"primary_shard_count" : 29,
"total_primary_bytes" : 169057528153
},
{
"version" : "7.17.10",
"index_count" : 33,
"primary_shard_count" : 33,
"total_primary_bytes" : 2445030685
}
]
},
"nodes" : {
"count" : {
"total" : 10,
"coordinating_only" : 0,
"data" : 10,
"data_cold" : 0,
"data_content" : 0,
"data_frozen" : 0,
"data_hot" : 0,
"data_warm" : 0,
"ingest" : 10,
"master" : 3,
"ml" : 0,
"remote_cluster_client" : 10,
"transform" : 10,
"voting_only" : 0
},
"versions" : [
"7.17.10"
],
"os" : {
"available_processors" : 400,
"allocated_processors" : 400,
"names" : [
{
"name" : "Linux",
"count" : 10
}
],
"pretty_names" : [
{
"pretty_name" : "CentOS Linux 7 (Core)",
"count" : 10
}
],
"architectures" : [
{
"arch" : "amd64",
"count" : 10
}
],
"mem" : {
"total_in_bytes" : 669656895488,
"free_in_bytes" : 10835079168,
"used_in_bytes" : 658821816320,
"free_percent" : 2,
"used_percent" : 98
}
},
"process" : {
"cpu" : {
"percent" : 242
},
"open_file_descriptors" : {
"min" : 803,
"max" : 895,
"avg" : 844
}
},
"jvm" : {
"max_uptime_in_millis" : 18680801258,
"versions" : [
{
"version" : "20.0.1",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "20.0.1+9-29",
"vm_vendor" : "Oracle Corporation",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"count" : 10
}
],
"mem" : {
"heap_used_in_bytes" : 90117027992,
"heap_max_in_bytes" : 257698037760
},
"threads" : 2292
},
"fs" : {
"total_in_bytes" : 3322859687936,
"free_in_bytes" : 2479345610752,
"available_in_bytes" : 2479345610752
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"security4" : 10
},
"http_types" : {
"security4" : 10
}
},
"discovery_types" : {
"zen" : 10
},
"packaging_types" : [
{
"flavor" : "default",
"type" : "rpm",
"count" : 10
}
],
"ingest" : {
"number_of_pipelines" : 2,
"processor_stats" : {
"gsub" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
},
"script" : {
"count" : 0,
"failed" : 0,
"current" : 0,
"time_in_millis" : 0
}
}
}
}
}