Hello,
We would like to set index.refresh_interval --> 120s as we are seeing write rejections on the data nodes.
But how to se this value on all the current indices (Fleet Managed) and the new indices which are going to create.
Quick help will be highly appreciated.
Thanks,
Praveen
If you are getting write rejections due to overloaded Elasticsearch then you probably have other issues you need to address. Generally though you would set this in the index template, otherwise here has an example you can adapt for your current indexes.
What is the output from the _cluster/stats?pretty&human API?
Just for future reference, there are no SLAs here and all responses are best effort. We can connect you with someone to discuss support options if you'd like? 
Thank you for response.
Here is the output i got when i ran the command _cluster/stats?pretty&human
{
"_nodes": {
"total": 11,
"successful": 11,
"failed": 0
},
"cluster_name": "by-prd-cluster",
"cluster_uuid": "Yf4vL2ZSR3elw_p-guIDgw",
"timestamp": 1674120367122,
"status": "green",
"indices": {
"count": 372,
"shards": {
"total": 744,
"primaries": 372,
"replication": 1,
"index": {
"shards": {
"min": 2,
"max": 2,
"avg": 2
},
"primaries": {
"min": 1,
"max": 1,
"avg": 1
},
"replication": {
"min": 1,
"max": 1,
"avg": 1
}
}
},
"docs": {
"count": 24288240260,
"deleted": 5317298
},
"store": {
"size": "18.3tb",
"size_in_bytes": 20187664479062,
"total_data_set_size": "18.3tb",
"total_data_set_size_in_bytes": 20187664479062,
"reserved": "0b",
"reserved_in_bytes": 0
},
"fielddata": {
"memory_size": "12.3mb",
"memory_size_in_bytes": 12969088,
"evictions": 0
},
"query_cache": {
"memory_size": "148.3mb",
"memory_size_in_bytes": 155586807,
"total_count": 537437445,
"hit_count": 33775437,
"miss_count": 503662008,
"cache_size": 32400,
"cache_count": 85264,
"evictions": 52864
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 16714,
"memory": "0b",
"memory_in_bytes": 0,
"terms_memory": "0b",
"terms_memory_in_bytes": 0,
"stored_fields_memory": "0b",
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "0b",
"norms_memory_in_bytes": 0,
"points_memory": "0b",
"points_memory_in_bytes": 0,
"doc_values_memory": "0b",
"doc_values_memory_in_bytes": 0,
"index_writer_memory": "132.8mb",
"index_writer_memory_in_bytes": 139268064,
"version_map_memory": "199.4kb",
"version_map_memory_in_bytes": 204249,
"fixed_bit_set": "3.1gb",
"fixed_bit_set_memory_in_bytes": 3372932808,
"max_unsafe_auto_id_timestamp": 1674116259374,
"file_sizes": {}
},
"mappings": {
"total_field_count": 132684,
"total_deduplicated_field_count": 24121,
"total_deduplicated_mapping_size": "166.6kb",
"total_deduplicated_mapping_size_in_bytes": 170655,
"field_types": [
{
"name": "alias",
"count": 43,
"index_count": 7,
"script_count": 0
},
{
"name": "binary",
"count": 5,
"index_count": 5,
"script_count": 0
},
{
"name": "boolean",
"count": 2093,
"index_count": 316,
"script_count": 0
},
{
"name": "byte",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "constant_keyword",
"count": 1280,
"index_count": 295,
"script_count": 0
},
{
"name": "date",
"count": 3272,
"index_count": 346,
"script_count": 0
},
{
"name": "date_nanos",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "date_range",
"count": 5,
"index_count": 5,
"script_count": 0
},
{
"name": "double",
"count": 193,
"index_count": 15,
"script_count": 0
},
{
"name": "double_range",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "flattened",
"count": 705,
"index_count": 136,
"script_count": 0
},
{
"name": "float",
"count": 2402,
"index_count": 164,
"script_count": 0
},
{
"name": "float_range",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "geo_point",
"count": 368,
"index_count": 195,
"script_count": 0
},
{
"name": "geo_shape",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "half_float",
"count": 57,
"index_count": 15,
"script_count": 0
},
{
"name": "integer",
"count": 191,
"index_count": 20,
"script_count": 0
},
{
"name": "integer_range",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "ip",
"count": 1096,
"index_count": 305,
"script_count": 0
},
{
"name": "ip_range",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "keyword",
"count": 60109,
"index_count": 346,
"script_count": 0
},
{
"name": "long",
"count": 25230,
"index_count": 311,
"script_count": 0
},
{
"name": "long_range",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "match_only_text",
"count": 517,
"index_count": 75,
"script_count": 0
},
{
"name": "nested",
"count": 668,
"index_count": 167,
"script_count": 0
},
{
"name": "object",
"count": 30664,
"index_count": 342,
"script_count": 0
},
{
"name": "scaled_float",
"count": 1519,
"index_count": 201,
"script_count": 0
},
{
"name": "shape",
"count": 1,
"index_count": 1,
"script_count": 0
},
{
"name": "short",
"count": 640,
"index_count": 14,
"script_count": 0
},
{
"name": "text",
"count": 1221,
"index_count": 295,
"script_count": 0
},
{
"name": "version",
"count": 8,
"index_count": 8,
"script_count": 0
},
{
"name": "wildcard",
"count": 389,
"index_count": 148,
"script_count": 0
}
],
"runtime_field_types": []
},
"analysis": {
"char_filter_types": [],
"tokenizer_types": [],
"filter_types": [],
"analyzer_types": [],
"built_in_char_filters": [],
"built_in_tokenizers": [],
"built_in_filters": [],
"built_in_analyzers": [
{
"name": "simple",
"count": 8,
"index_count": 4
}
]
},
"versions": [
{
"version": "7.14.0",
"index_count": 112,
"primary_shard_count": 112,
"total_primary_size": "1.7tb",
"total_primary_bytes": 1874055566292
},
{
"version": "7.17.6",
"index_count": 5,
"primary_shard_count": 5,
"total_primary_size": "244.7mb",
"total_primary_bytes": 256663881
},
{
"version": "8.4.1",
"index_count": 255,
"primary_shard_count": 255,
"total_primary_size": "7.4tb",
"total_primary_bytes": 8218466702945
}
]
},
"nodes": {
"count": {
"total": 11,
"coordinating_only": 0,
"data": 5,
"data_cold": 0,
"data_content": 0,
"data_frozen": 0,
"data_hot": 0,
"data_warm": 0,
"ingest": 3,
"master": 3,
"ml": 0,
"remote_cluster_client": 0,
"transform": 0,
"voting_only": 0
},
"versions": [
"8.4.1"
],
"os": {
"available_processors": 32,
"allocated_processors": 32,
"names": [
{
"name": "Linux",
"count": 11
}
],
"pretty_names": [
{
"pretty_name": "Red Hat Enterprise Linux 8.4 (Ootpa)",
"count": 11
}
],
"architectures": [
{
"arch": "amd64",
"count": 11
}
],
"mem": {
"total": "187.1gb",
"total_in_bytes": 200917213184,
"adjusted_total": "187.1gb",
"adjusted_total_in_bytes": 200917213184,
"free": "32.3gb",
"free_in_bytes": 34717655040,
"used": "154.7gb",
"used_in_bytes": 166199558144,
"free_percent": 17,
"used_percent": 83
}
},
"process": {
"cpu": {
"percent": 451
},
"open_file_descriptors": {
"min": 620,
"max": 3747,
"avg": 2717
}
},
"jvm": {
"max_uptime": "121.9d",
"max_uptime_in_millis": 10538766231,
"versions": [
{
"version": "18.0.2",
"vm_name": "OpenJDK 64-Bit Server VM",
"vm_version": "18.0.2+9-61",
"vm_vendor": "Oracle Corporation",
"bundled_jdk": true,
"using_bundled_jdk": true,
"count": 11
}
],
"mem": {
"heap_used": "16.7gb",
"heap_used_in_bytes": 17981596704,
"heap_max": "32gb",
"heap_max_in_bytes": 34359738368
},
"threads": 886
},
"fs": {
"total": "29.9tb",
"total_in_bytes": 32912676270080,
"free": "11.3tb",
"free_in_bytes": 12455265697792,
"available": "11.3tb",
"available_in_bytes": 12455265697792
},
"plugins": [],
"network_types": {
"transport_types": {
"security4": 11
},
"http_types": {
"security4": 11
}
},
"discovery_types": {
"multi-node": 11
},
"packaging_types": [
{
"flavor": "default",
"type": "rpm",
"count": 11
}
],
"ingest": {
"number_of_pipelines": 313,
"processor_stats": {
"append": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"community_id": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"conditional": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"convert": {
"count": 258,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"csv": {
"count": 86,
"failed": 0,
"current": 0,
"time": "4ms",
"time_in_millis": 4
},
"date": {
"count": 21414279809,
"failed": 0,
"current": 0,
"time": "2.6d",
"time_in_millis": 228313889
},
"dissect": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"dot_expander": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"fingerprint": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"foreach": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"geoip": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"grok": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"gsub": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"join": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"json": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"kv": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"lowercase": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"network_direction": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"pipeline": {
"count": 21487714443,
"failed": 0,
"current": 0,
"time": "106751.9d",
"time_in_millis": 98095075789620
},
"remove": {
"count": 42828559532,
"failed": 0,
"current": 0,
"time": "6.8h",
"time_in_millis": 24830591
},
"rename": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"script": {
"count": 21414279723,
"failed": 0,
"current": 0,
"time": "10.2h",
"time_in_millis": 36750510
},
"set": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"set_security_user": {
"count": 21414279723,
"failed": 0,
"current": 0,
"time": "23.4h",
"time_in_millis": 84426389
},
"split": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"trim": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"uppercase": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"uri_parts": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"urldecode": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
},
"user_agent": {
"count": 0,
"failed": 0,
"current": 0,
"time": "0s",
"time_in_millis": 0
}
}
},
"indexing_pressure": {
"memory": {
"current": {
"combined_coordinating_and_primary": "0b",
"combined_coordinating_and_primary_in_bytes": 0,
"coordinating": "0b",
"coordinating_in_bytes": 0,
"primary": "0b",
"primary_in_bytes": 0,
"replica": "0b",
"replica_in_bytes": 0,
"all": "0b",
"all_in_bytes": 0
},
"total": {
"combined_coordinating_and_primary": "0b",
"combined_coordinating_and_primary_in_bytes": 0,
"coordinating": "0b",
"coordinating_in_bytes": 0,
"primary": "0b",
"primary_in_bytes": 0,
"replica": "0b",
"replica_in_bytes": 0,
"all": "0b",
"all_in_bytes": 0,
"coordinating_rejections": 0,
"primary_rejections": 0,
"replica_rejections": 0
},
"limit": "0b",
"limit_in_bytes": 0
}
}
}
}
What size heap do you have on your nodes, it looks like you have 32GB across 11 nodes?
For data nodes we have set Heap Size as 4G and for Ingest & master nodes Heap Size is 2G.
Master Nodes (VM count=3) --> 8GB RAM each
Data Nodes (VM count=5) --> 24 GB RAM each
Ingest Nodes (VM count=5) --> 16 GB RAM each
Kibana (VM Count=1) --> 16 GB RAM.
Please let me know if you need any other information.
@warkolm can you please validate the data i have provided.
Thanks,
Praveen
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.
© 2020. All Rights Reserved - Elasticsearch
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant logo are trademarks of the Apache Software Foundation in the United States and/or other countries.