Search rate and index latency is high

I'm using elasticsearch as database but I'm facing momentary surge in search rate and latency. Can anyone tell why there is a high search rate and latency observerd I have 10 nodes of cluster which are physical vms. It's working fine as of now but and I'm putting the below stats of cluster:

{
  "_nodes" : {
    "total" : 10,
    "successful" : 10,
    "failed" : 0
  },
  "cluster_name" : "graphreadz2vs",
  "cluster_uuid" : "D63jtD0hahgeuk3js8ja9dw2mEdGSw",
  "timestamp" : 1714392526175,
  "status" : "green",
  "indices" : {
    "count" : 39,
    "shards" : {
      "total" : 197,
      "primaries" : 62,
      "replication" : 2.1774193548387095,
      "index" : {
        "shards" : {
          "min" : 2,
          "max" : 72,
          "avg" : 5.051282051282051
        },
        "primaries" : {
          "min" : 1,
          "max" : 9,
          "avg" : 1.5897435897435896
        },
        "replication" : {
          "min" : 1.0,
          "max" : 7.0,
          "avg" : 1.2564102564102564
        }
      }
    },
    "docs" : {
      "count" : 619585818,
      "deleted" : 118392189
    },
    "store" : {
      "size_in_bytes" : 838169390789,
      "total_data_set_size_in_bytes" : 838169390789,
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size_in_bytes" : 14112,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size_in_bytes" : 5441447555,
      "total_count" : 3109663468713,
      "hit_count" : 122009856826,
      "miss_count" : 2987653611887,
      "cache_size" : 986242,
      "cache_count" : 5177072,
      "evictions" : 4190830
    },
    "completion" : {
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 2780,
      "memory_in_bytes" : 42113584,
      "terms_memory_in_bytes" : 22388376,
      "stored_fields_memory_in_bytes" : 6534432,
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory_in_bytes" : 367296,
      "points_memory_in_bytes" : 0,
      "doc_values_memory_in_bytes" : 12823480,
      "index_writer_memory_in_bytes" : 3372616,
      "version_map_memory_in_bytes" : 3031,
      "fixed_bit_set_memory_in_bytes" : 655860408,
      "max_unsafe_auto_id_timestamp" : 1714371858260,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 55,
          "index_count" : 21,
          "script_count" : 0
        },
        {
          "name" : "constant_keyword",
          "count" : 6,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "date",
          "count" : 128,
          "index_count" : 30,
          "script_count" : 0
        },
        {
          "name" : "float",
          "count" : 83,
          "index_count" : 13,
          "script_count" : 0
        },
        {
          "name" : "half_float",
          "count" : 56,
          "index_count" : 14,
          "script_count" : 0
        },
        {
          "name" : "integer",
          "count" : 179,
          "index_count" : 11,
          "script_count" : 0
        },
        {
          "name" : "ip",
          "count" : 2,
          "index_count" : 2,
          "script_count" : 0
        },
        {
          "name" : "keyword",
          "count" : 807,
          "index_count" : 30,
          "script_count" : 0
        },
        {
          "name" : "long",
          "count" : 1325,
          "index_count" : 25,
          "script_count" : 0
        },
        {
          "name" : "nested",
          "count" : 33,
          "index_count" : 13,
          "script_count" : 0
        },
        {
          "name" : "object",
          "count" : 885,
          "index_count" : 25,
          "script_count" : 0
        },
        {
          "name" : "text",
          "count" : 103,
          "index_count" : 20,
          "script_count" : 0
        },
        {
          "name" : "version",
          "count" : 4,
          "index_count" : 4,
          "script_count" : 0
        }
      ],
      "runtime_field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.8.1",
        "index_count" : 6,
        "primary_shard_count" : 29,
        "total_primary_bytes" : 169057528153
      },
      {
        "version" : "7.17.10",
        "index_count" : 33,
        "primary_shard_count" : 33,
        "total_primary_bytes" : 2445030685
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 10,
      "coordinating_only" : 0,
      "data" : 10,
      "data_cold" : 0,
      "data_content" : 0,
      "data_frozen" : 0,
      "data_hot" : 0,
      "data_warm" : 0,
      "ingest" : 10,
      "master" : 3,
      "ml" : 0,
      "remote_cluster_client" : 10,
      "transform" : 10,
      "voting_only" : 0
    },
    "versions" : [
      "7.17.10"
    ],
    "os" : {
      "available_processors" : 400,
      "allocated_processors" : 400,
      "names" : [
        {
          "name" : "Linux",
          "count" : 10
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 7 (Core)",
          "count" : 10
        }
      ],
      "architectures" : [
        {
          "arch" : "amd64",
          "count" : 10
        }
      ],
      "mem" : {
        "total_in_bytes" : 669656895488,
        "free_in_bytes" : 10835079168,
        "used_in_bytes" : 658821816320,
        "free_percent" : 2,
        "used_percent" : 98
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 242
      },
      "open_file_descriptors" : {
        "min" : 803,
        "max" : 895,
        "avg" : 844
      }
    },
    "jvm" : {
      "max_uptime_in_millis" : 18680801258,
      "versions" : [
        {
          "version" : "20.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "20.0.1+9-29",
          "vm_vendor" : "Oracle Corporation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 10
        }
      ],
      "mem" : {
        "heap_used_in_bytes" : 90117027992,
        "heap_max_in_bytes" : 257698037760
      },
      "threads" : 2292
    },
    "fs" : {
      "total_in_bytes" : 3322859687936,
      "free_in_bytes" : 2479345610752,
      "available_in_bytes" : 2479345610752
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 10
      },
      "http_types" : {
        "security4" : 10
      }
    },
    "discovery_types" : {
      "zen" : 10
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 10
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 2,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time_in_millis" : 0
        }
      }
    }
  }
}
1 Like

I have the same problem but on my server. I am running elk in separate containers but on the same network and I think this can be issue. Disable swapping | Elasticsearch Guide [8.13] | Elastic I am still examining my case.

Swap is already disabled