Elasticsearch performance

Elastic Stack Elasticsearch
1

I have an issue when try to update in Elasticsearch, sometimes it takes more than 10 seconds
According to the elastic logs, the overall health check itself takes too long , below example is > 20 seconds

{"type": "server", "timestamp": "2022-03-31T14:07:56,788Z", "level": "WARN", "component": "o.e.m.f.FsHealthService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-0", "message": "health check of [/usr/share/elasticsearch/data/nodes/0] took [10605ms] which is above the warn threshold of [5s]", "cluster.uuid": "XcF8SobFT5WfznloIQOmFQ", "node.id": "Np6OgulyR6mjljoRrLoRHg" }
I deploy Elasticsearch on Openshift Redhat and I used persistent volume to save my data there

2

What is the output from the _cluster/stats?pretty&human API?

3

check below the output from _cluster/stats?pretty&human

{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "XcF8SobFT5WfznloIQOmFQ",
  "timestamp" : 1649057229947,
  "status" : "yellow",
  "indices" : {
    "count" : 14,
    "shards" : {
      "total" : 14,
      "primaries" : 14,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 1,
          "avg" : 1.0
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 38665346,
      "deleted" : 7
    },
    "store" : {
      "size" : "1.4gb",
      "size_in_bytes" : 1596997137,
      "total_data_set_size" : "1.4gb",
      "total_data_set_size_in_bytes" : 1596997137,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "14.7kb",
      "memory_size_in_bytes" : 15064,
      "total_count" : 600,
      "hit_count" : 0,
      "miss_count" : 600,
      "cache_size" : 1,
      "cache_count" : 2,
      "evictions" : 1
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 75,
      "memory" : "375.9kb",
      "memory_in_bytes" : 385020,
      "terms_memory" : "312.4kb",
      "terms_memory_in_bytes" : 319968,
      "stored_fields_memory" : "39kb",
      "stored_fields_memory_in_bytes" : 39992,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "6kb",
      "norms_memory_in_bytes" : 6144,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "18.4kb",
      "doc_values_memory_in_bytes" : 18916,
      "index_writer_memory" : "1.2mb",
      "index_writer_memory_in_bytes" : 1271240,
      "version_map_memory" : "127b",
      "version_map_memory_in_bytes" : 127,
      "fixed_bit_set" : "144.9kb",
      "fixed_bit_set_memory_in_bytes" : 148464,
      "max_unsafe_auto_id_timestamp" : -1,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [
        {
          "name" : "boolean",
          "count" : 6,
          "index_count" : 4,
          "script_count" : 0
        },
        {
          "name" : "date",
          "count" : 6,
          "index_count" : 4,
          "script_count" : 0
        },
        {
          "name" : "keyword",
          "count" : 293,
          "index_count" : 14,
          "script_count" : 0
        },
        {
          "name" : "long",
          "count" : 21,
          "index_count" : 7,
          "script_count" : 0
        },
        {
          "name" : "nested",
          "count" : 8,
          "index_count" : 4,
          "script_count" : 0
        },
        {
          "name" : "object",
          "count" : 20,
          "index_count" : 11,
          "script_count" : 0
        },
        {
          "name" : "text",
          "count" : 23,
          "index_count" : 8,
          "script_count" : 0
        }
      ],
      "runtime_field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [
        {
          "name" : "edge_ngram",
          "count" : 12,
          "index_count" : 12
        }
      ],
      "filter_types" : [ ],
      "analyzer_types" : [
        {
          "name" : "custom",
          "count" : 12,
          "index_count" : 12
        }
      ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [
        {
          "name" : "lowercase",
          "count" : 12,
          "index_count" : 12
        }
      ],
      "built_in_analyzers" : [ ]
    },
    "versions" : [
      {
        "version" : "7.14.1",
        "index_count" : 14,
        "primary_shard_count" : 14,
        "total_primary_size" : "1.4gb",
        "total_primary_bytes" : 1596997137
      }
    ]
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "data_cold" : 1,
      "data_content" : 1,
      "data_frozen" : 1,
      "data_hot" : 1,
      "data_warm" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "remote_cluster_client" : 1,
      "transform" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.14.1"
    ],
    "os" : {
      "available_processors" : 2,
      "allocated_processors" : 2,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "CentOS Linux 8",
          "count" : 1
        }
      ],
      "architectures" : [
        {
          "arch" : "amd64",
          "count" : 1
        }
      ],
      "mem" : {
        "total" : "4gb",
        "total_in_bytes" : 4294967296,
        "free" : "748.1mb",
        "free_in_bytes" : 784465920,
        "used" : "3.2gb",
        "used_in_bytes" : 3510501376,
        "free_percent" : 18,
        "used_percent" : 82
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 0
      },
      "open_file_descriptors" : {
        "min" : 369,
        "max" : 369,
        "avg" : 369
      }
    },
    "jvm" : {
      "max_uptime" : "3.7d",
      "max_uptime_in_millis" : 325067570,
      "versions" : [
        {
          "version" : "16.0.2",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "16.0.2+7",
          "vm_vendor" : "Eclipse Foundation",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used" : "652.4mb",
        "heap_used_in_bytes" : 684124624,
        "heap_max" : "2gb",
        "heap_max_in_bytes" : 2147483648
      },
      "threads" : 53
    },
    "fs" : {
      "total" : "3.8gb",
      "total_in_bytes" : 4160421888,
      "free" : "2.3gb",
      "free_in_bytes" : 2544877568,
      "available" : "2.3gb",
      "available_in_bytes" : 2528100352
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "zen" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "docker",
        "count" : 1
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 1,
      "processor_stats" : {
        "gsub" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        },
        "script" : {
          "count" : 0,
          "failed" : 0,
          "current" : 0,
          "time" : "0s",
          "time_in_millis" : 0
        }
      }
    }
  }
}
4

Thanks.

What sort of storage backend is it?

5

I don't know, how I get this information, I am deploying Elasticsearch on Openshift

6

You need to check with someone that administrates OpenShift what type of storage that is used. It could be that it is some type of slow networked storage that is not really suitable for Elasticsearch, which can be very I/O intensive and require fast disks.

7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.