How to change "index.refresh_interval" value

Hello,

We would like to set index.refresh_interval --> 120s as we are seeing write rejections on the data nodes.

But how to se this value on all the current indices (Fleet Managed) and the new indices which are going to create.

Quick help will be highly appreciated.

Thanks,
Praveen

If you are getting write rejections due to overloaded Elasticsearch then you probably have other issues you need to address. Generally though you would set this in the index template, otherwise here has an example you can adapt for your current indexes.

What is the output from the _cluster/stats?pretty&human API?

Just for future reference, there are no SLAs here and all responses are best effort. We can connect you with someone to discuss support options if you'd like? :slight_smile:

Thank you for response.

Here is the output i got when i ran the command _cluster/stats?pretty&human

{
  "_nodes": {
    "total": 11,
    "successful": 11,
    "failed": 0
  },
  "cluster_name": "by-prd-cluster",
  "cluster_uuid": "Yf4vL2ZSR3elw_p-guIDgw",
  "timestamp": 1674120367122,
  "status": "green",
  "indices": {
    "count": 372,
    "shards": {
      "total": 744,
      "primaries": 372,
      "replication": 1,
      "index": {
        "shards": {
          "min": 2,
          "max": 2,
          "avg": 2
        },
        "primaries": {
          "min": 1,
          "max": 1,
          "avg": 1
        },
        "replication": {
          "min": 1,
          "max": 1,
          "avg": 1
        }
      }
    },
    "docs": {
      "count": 24288240260,
      "deleted": 5317298
    },
    "store": {
      "size": "18.3tb",
      "size_in_bytes": 20187664479062,
      "total_data_set_size": "18.3tb",
      "total_data_set_size_in_bytes": 20187664479062,
      "reserved": "0b",
      "reserved_in_bytes": 0
    },
    "fielddata": {
      "memory_size": "12.3mb",
      "memory_size_in_bytes": 12969088,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "148.3mb",
      "memory_size_in_bytes": 155586807,
      "total_count": 537437445,
      "hit_count": 33775437,
      "miss_count": 503662008,
      "cache_size": 32400,
      "cache_count": 85264,
      "evictions": 52864
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 16714,
      "memory": "0b",
      "memory_in_bytes": 0,
      "terms_memory": "0b",
      "terms_memory_in_bytes": 0,
      "stored_fields_memory": "0b",
      "stored_fields_memory_in_bytes": 0,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "0b",
      "norms_memory_in_bytes": 0,
      "points_memory": "0b",
      "points_memory_in_bytes": 0,
      "doc_values_memory": "0b",
      "doc_values_memory_in_bytes": 0,
      "index_writer_memory": "132.8mb",
      "index_writer_memory_in_bytes": 139268064,
      "version_map_memory": "199.4kb",
      "version_map_memory_in_bytes": 204249,
      "fixed_bit_set": "3.1gb",
      "fixed_bit_set_memory_in_bytes": 3372932808,
      "max_unsafe_auto_id_timestamp": 1674116259374,
      "file_sizes": {}
    },
    "mappings": {
      "total_field_count": 132684,
      "total_deduplicated_field_count": 24121,
      "total_deduplicated_mapping_size": "166.6kb",
      "total_deduplicated_mapping_size_in_bytes": 170655,
      "field_types": [
        {
          "name": "alias",
          "count": 43,
          "index_count": 7,
          "script_count": 0
        },
        {
          "name": "binary",
          "count": 5,
          "index_count": 5,
          "script_count": 0
        },
        {
          "name": "boolean",
          "count": 2093,
          "index_count": 316,
          "script_count": 0
        },
        {
          "name": "byte",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "constant_keyword",
          "count": 1280,
          "index_count": 295,
          "script_count": 0
        },
        {
          "name": "date",
          "count": 3272,
          "index_count": 346,
          "script_count": 0
        },
        {
          "name": "date_nanos",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "date_range",
          "count": 5,
          "index_count": 5,
          "script_count": 0
        },
        {
          "name": "double",
          "count": 193,
          "index_count": 15,
          "script_count": 0
        },
        {
          "name": "double_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "flattened",
          "count": 705,
          "index_count": 136,
          "script_count": 0
        },
        {
          "name": "float",
          "count": 2402,
          "index_count": 164,
          "script_count": 0
        },
        {
          "name": "float_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "geo_point",
          "count": 368,
          "index_count": 195,
          "script_count": 0
        },
        {
          "name": "geo_shape",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "half_float",
          "count": 57,
          "index_count": 15,
          "script_count": 0
        },
        {
          "name": "integer",
          "count": 191,
          "index_count": 20,
          "script_count": 0
        },
        {
          "name": "integer_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "ip",
          "count": 1096,
          "index_count": 305,
          "script_count": 0
        },
        {
          "name": "ip_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "keyword",
          "count": 60109,
          "index_count": 346,
          "script_count": 0
        },
        {
          "name": "long",
          "count": 25230,
          "index_count": 311,
          "script_count": 0
        },
        {
          "name": "long_range",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "match_only_text",
          "count": 517,
          "index_count": 75,
          "script_count": 0
        },
        {
          "name": "nested",
          "count": 668,
          "index_count": 167,
          "script_count": 0
        },
        {
          "name": "object",
          "count": 30664,
          "index_count": 342,
          "script_count": 0
        },
        {
          "name": "scaled_float",
          "count": 1519,
          "index_count": 201,
          "script_count": 0
        },
        {
          "name": "shape",
          "count": 1,
          "index_count": 1,
          "script_count": 0
        },
        {
          "name": "short",
          "count": 640,
          "index_count": 14,
          "script_count": 0
        },
        {
          "name": "text",
          "count": 1221,
          "index_count": 295,
          "script_count": 0
        },
        {
          "name": "version",
          "count": 8,
          "index_count": 8,
          "script_count": 0
        },
        {
          "name": "wildcard",
          "count": 389,
          "index_count": 148,
          "script_count": 0
        }
      ],
      "runtime_field_types": []
    },
    "analysis": {
      "char_filter_types": [],
      "tokenizer_types": [],
      "filter_types": [],
      "analyzer_types": [],
      "built_in_char_filters": [],
      "built_in_tokenizers": [],
      "built_in_filters": [],
      "built_in_analyzers": [
        {
          "name": "simple",
          "count": 8,
          "index_count": 4
        }
      ]
    },
    "versions": [
      {
        "version": "7.14.0",
        "index_count": 112,
        "primary_shard_count": 112,
        "total_primary_size": "1.7tb",
        "total_primary_bytes": 1874055566292
      },
      {
        "version": "7.17.6",
        "index_count": 5,
        "primary_shard_count": 5,
        "total_primary_size": "244.7mb",
        "total_primary_bytes": 256663881
      },
      {
        "version": "8.4.1",
        "index_count": 255,
        "primary_shard_count": 255,
        "total_primary_size": "7.4tb",
        "total_primary_bytes": 8218466702945
      }
    ]
  },
  "nodes": {
    "count": {
      "total": 11,
      "coordinating_only": 0,
      "data": 5,
      "data_cold": 0,
      "data_content": 0,
      "data_frozen": 0,
      "data_hot": 0,
      "data_warm": 0,
      "ingest": 3,
      "master": 3,
      "ml": 0,
      "remote_cluster_client": 0,
      "transform": 0,
      "voting_only": 0
    },
    "versions": [
      "8.4.1"
    ],
    "os": {
      "available_processors": 32,
      "allocated_processors": 32,
      "names": [
        {
          "name": "Linux",
          "count": 11
        }
      ],
      "pretty_names": [
        {
          "pretty_name": "Red Hat Enterprise Linux 8.4 (Ootpa)",
          "count": 11
        }
      ],
      "architectures": [
        {
          "arch": "amd64",
          "count": 11
        }
      ],
      "mem": {
        "total": "187.1gb",
        "total_in_bytes": 200917213184,
        "adjusted_total": "187.1gb",
        "adjusted_total_in_bytes": 200917213184,
        "free": "32.3gb",
        "free_in_bytes": 34717655040,
        "used": "154.7gb",
        "used_in_bytes": 166199558144,
        "free_percent": 17,
        "used_percent": 83
      }
    },
    "process": {
      "cpu": {
        "percent": 451
      },
      "open_file_descriptors": {
        "min": 620,
        "max": 3747,
        "avg": 2717
      }
    },
    "jvm": {
      "max_uptime": "121.9d",
      "max_uptime_in_millis": 10538766231,
      "versions": [
        {
          "version": "18.0.2",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "18.0.2+9-61",
          "vm_vendor": "Oracle Corporation",
          "bundled_jdk": true,
          "using_bundled_jdk": true,
          "count": 11
        }
      ],
      "mem": {
        "heap_used": "16.7gb",
        "heap_used_in_bytes": 17981596704,
        "heap_max": "32gb",
        "heap_max_in_bytes": 34359738368
      },
      "threads": 886
    },
    "fs": {
      "total": "29.9tb",
      "total_in_bytes": 32912676270080,
      "free": "11.3tb",
      "free_in_bytes": 12455265697792,
      "available": "11.3tb",
      "available_in_bytes": 12455265697792
    },
    "plugins": [],
    "network_types": {
      "transport_types": {
        "security4": 11
      },
      "http_types": {
        "security4": 11
      }
    },
    "discovery_types": {
      "multi-node": 11
    },
    "packaging_types": [
      {
        "flavor": "default",
        "type": "rpm",
        "count": 11
      }
    ],
    "ingest": {
      "number_of_pipelines": 313,
      "processor_stats": {
        "append": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "community_id": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "conditional": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "convert": {
          "count": 258,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "csv": {
          "count": 86,
          "failed": 0,
          "current": 0,
          "time": "4ms",
          "time_in_millis": 4
        },
        "date": {
          "count": 21414279809,
          "failed": 0,
          "current": 0,
          "time": "2.6d",
          "time_in_millis": 228313889
        },
        "dissect": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "dot_expander": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "fingerprint": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "foreach": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "geoip": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "grok": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "gsub": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "join": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "json": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "kv": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "lowercase": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "network_direction": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "pipeline": {
          "count": 21487714443,
          "failed": 0,
          "current": 0,
          "time": "106751.9d",
          "time_in_millis": 98095075789620
        },
        "remove": {
          "count": 42828559532,
          "failed": 0,
          "current": 0,
          "time": "6.8h",
          "time_in_millis": 24830591
        },
        "rename": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "script": {
          "count": 21414279723,
          "failed": 0,
          "current": 0,
          "time": "10.2h",
          "time_in_millis": 36750510
        },
        "set": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "set_security_user": {
          "count": 21414279723,
          "failed": 0,
          "current": 0,
          "time": "23.4h",
          "time_in_millis": 84426389
        },
        "split": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "trim": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "uppercase": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "uri_parts": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "urldecode": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        },
        "user_agent": {
          "count": 0,
          "failed": 0,
          "current": 0,
          "time": "0s",
          "time_in_millis": 0
        }
      }
    },
    "indexing_pressure": {
      "memory": {
        "current": {
          "combined_coordinating_and_primary": "0b",
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating": "0b",
          "coordinating_in_bytes": 0,
          "primary": "0b",
          "primary_in_bytes": 0,
          "replica": "0b",
          "replica_in_bytes": 0,
          "all": "0b",
          "all_in_bytes": 0
        },
        "total": {
          "combined_coordinating_and_primary": "0b",
          "combined_coordinating_and_primary_in_bytes": 0,
          "coordinating": "0b",
          "coordinating_in_bytes": 0,
          "primary": "0b",
          "primary_in_bytes": 0,
          "replica": "0b",
          "replica_in_bytes": 0,
          "all": "0b",
          "all_in_bytes": 0,
          "coordinating_rejections": 0,
          "primary_rejections": 0,
          "replica_rejections": 0
        },
        "limit": "0b",
        "limit_in_bytes": 0
      }
    }
  }
}

What size heap do you have on your nodes, it looks like you have 32GB across 11 nodes?

For data nodes we have set Heap Size as 4G and for Ingest & master nodes Heap Size is 2G.

Master Nodes (VM count=3) --> 8GB RAM each

Data Nodes (VM count=5) --> 24 GB RAM each

Ingest Nodes (VM count=5) --> 16 GB RAM each

Kibana (VM Count=1) --> 16 GB RAM.

Please let me know if you need any other information.

@warkolm can you please validate the data i have provided.

Thanks,
Praveen