Routing allocation shard doesn't work

Elastic Stack Elasticsearch
1

Hi
I'm struggling the case why even it was enabled the routing rebalance on that index.
shards are still placed on one node, how to force automatically rerouting shards on the other nodes?
some of the index(with shards) are occupied only one of 5 nodes?? why??
I don't want to make it one by one for reroute, I thought that elastic take decisions byself

logstash-system_log-003147           3     p      STARTED     72130482 es_data_ssd_1_3
logstash-system_log-003147           1     p      STARTED     72140842 es_data_ssd_1_3
logstash-system_log-003147           4     p      STARTED     72142868 es_data_ssd_1_3
logstash-system_log-003147           2     p      STARTED     72155930 es_data_ssd_1_3
logstash-system_log-003147           0     p      STARTED     72151212 es_data_ssd_1_3

 "routing" : {
          "rebalance" : {
            "enable" : "all"
          },
          "allocation" : {
            "disk" : {
              "watermark" : {
                "ignore" : "false"
              }
            },
            "enable" : "all",
            "total_shards_per_node" : "-1"
2

To get an Elasticsearch cluster to explain why it's allocating shards as it is, use the cluster allocation explain API. If you need help understanding its output, share it (in full) here

3 
{
  "index" : "logstash-system_log-003147",
  "shard" : 0,
  "primary" : true,
  "current_state" : "started",
  "current_node" : {
    "id" : "azCWBxpOTnC9Tj5vZSw1yw",
    "name" : "es_data_ssd_1_3",
    "transport_address" : "10.0.9.118:9300",
    "attributes" : {
      "rack_id" : "rack_three",
      "xpack.installed" : "true"
    },
    "weight_ranking" : 1
  },
  "can_remain_on_current_node" : "yes",
  "can_rebalance_cluster" : "throttled",
  "can_rebalance_cluster_decisions" : [
    {
      "decider" : "concurrent_rebalance",
      "decision" : "THROTTLE",
      "explanation" : "reached the limit of concurrently rebalancing shards [60], cluster setting [cluster.routing.allocation.cluster_concurrent_rebalance=2]"
    }
  ],
  "can_rebalance_to_other_node" : "throttled",
  "rebalance_explanation" : "rebalancing is throttled",
  "node_allocation_decisions" : [
    {
      "node_id" : "AOKUKtaSS0anhJGZrQrCUA",
      "node_name" : "es_data_hdd_4_3",
      "transport_address" : "10.0.9.133:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 16,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "l_Yfxkq4QHKFRnlgEM9iiw",
      "node_name" : "es_data_hdd_7_1",
      "transport_address" : "10.0.9.93:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 17,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "qUitJPvbSs2I7MT3sXSCYA",
      "node_name" : "es_data_hdd_7_2",
      "transport_address" : "10.0.9.112:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 18,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "Xz2QXNZJSrSMB-HMWXXt0g",
      "node_name" : "es_data_hdd_4_2",
      "transport_address" : "10.0.9.110:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 19,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "u-ssDIvjRsCGnJSU23EiDA",
      "node_name" : "es_data_hdd_1_1",
      "transport_address" : "10.0.9.89:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 20,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "WStEXKOmT4WLUKYgXDgd9w",
      "node_name" : "es_data_hdd_4_1",
      "transport_address" : "10.0.9.83:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 21,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "c0JyzTbrTq2tVtsIfg7rwg",
      "node_name" : "es_data_hdd_1_2",
      "transport_address" : "10.0.9.108:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 22,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "4xdIx8gtR9-YZWXR_WuwHw",
      "node_name" : "es_data_hdd_7_3",
      "transport_address" : "10.0.9.121:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 23,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "GWiGvV_kRja2AS6lRBDquQ",
      "node_name" : "es_data_hdd_1_3",
      "transport_address" : "10.0.9.137:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "no",
      "weight_ranking" : 24,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of incoming shard recoveries [6], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        },
        {
          "decider" : "data_tier",
          "decision" : "NO",
          "explanation" : "index has a preference for tiers [data_content] and node does not meet the required [data_content] tier"
        }
      ]
    },
    {
      "node_id" : "hNM_5KXbQM2UtfbgXq6maA",
      "node_name" : "es_data_ssd_3_1",
      "transport_address" : "10.0.9.87:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 2,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "fI-rzQY5Q4qgEF8vozmJzw",
      "node_name" : "es_data_ssd_1_1",
      "transport_address" : "10.0.9.85:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 3,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "XMyLrsVFSJiKSs-yUhZkgA",
      "node_name" : "es_data_ssd_5_1",
      "transport_address" : "10.0.9.79:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 4,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "DsIvBrBQTjW0F8h8E-q-0g",
      "node_name" : "es_data_ssd_4_1",
      "transport_address" : "10.0.9.81:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 5,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "p42W5U35QRycbkC_n3zvJQ",
      "node_name" : "es_data_ssd_2_1",
      "transport_address" : "10.0.9.75:9300",
      "node_attributes" : {
        "rack_id" : "rack_one",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 6,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "eRk6tLc3RFG0zYOsnFPFUw",
      "node_name" : "es_data_ssd_4_3",
      "transport_address" : "10.0.9.135:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 7,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "WqnNe05MTuuYzY8uwI_P7Q",
      "node_name" : "es_data_ssd_5_3",
      "transport_address" : "10.0.9.125:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 8,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "8RZ_g_qIQjSW7g_jgA0LRA",
      "node_name" : "es_data_ssd_3_3",
      "transport_address" : "10.0.9.139:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 9,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "YEzEpdGpT7iECEXqVTVhDQ",
      "node_name" : "es_data_ssd_2_3",
      "transport_address" : "10.0.9.129:9300",
      "node_attributes" : {
        "rack_id" : "rack_three",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 10,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "fjWTi2hUQxO3tEzT5zwGog",
      "node_name" : "es_data_ssd_3_2",
      "transport_address" : "10.0.9.98:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 11,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "NPcmoyidSr2r4GiO07uimw",
      "node_name" : "es_data_ssd_4_2",
      "transport_address" : "10.0.9.114:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 12,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "ifoM86KLRxarQqBVnAoa4A",
      "node_name" : "es_data_ssd_5_2",
      "transport_address" : "10.0.9.102:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 13,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "wgCiK7OqTbG6XUPTtv-_gg",
      "node_name" : "es_data_ssd_2_2",
      "transport_address" : "10.0.9.106:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 14,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    },
    {
      "node_id" : "1YxZtfHETGqHHxfbVI3lHQ",
      "node_name" : "es_data_ssd_1_2",
      "transport_address" : "10.0.9.95:9300",
      "node_attributes" : {
        "rack_id" : "rack_two",
        "xpack.installed" : "true"
      },
      "node_decision" : "worse_balance",
      "weight_ranking" : 15,
      "deciders" : [
        {
          "decider" : "throttling",
          "decision" : "THROTTLE",
          "explanation" : "reached the limit of outgoing shard recoveries [6] on the node [azCWBxpOTnC9Tj5vZSw1yw] which holds the primary, cluster setting [cluster.routing.allocation.node_concurrent_outgoing_recoveries=6] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])"
        }
      ]
    }
  ]
}
4

now I have change from 2 to 6 <cluster.routing.allocation.cluster_concurrent_rebalance>

{
  "persistent" : {
    "cluster.max_shards_per_node" : "1500",
    "cluster.routing.allocation.allow_rebalance" : "always",
    "cluster.routing.allocation.disk.watermark.flood_stage" : "97%",
    "cluster.routing.allocation.disk.watermark.high" : "95%",
    "cluster.routing.allocation.disk.watermark.low" : "90%",
    "cluster.routing.allocation.enable" : "all",
    "cluster.routing.allocation.exclude._name" : "null",
    "cluster.routing.allocation.node_concurrent_recoveries" : "6",
    "cluster.routing.rebalance.enable" : "all",
    "indices.breaker.request.limit" : "98%",
    "indices.breaker.total.limit" : "98%",
    "indices.recovery.max_concurrent_file_chunks" : "3",
    "xpack.monitoring.collection.enabled" : "true",
    "xpack.monitoring.elasticsearch.collection.enabled" : "true"
  },
  "transient" : {
    "cluster.routing.allocation.cluster_concurrent_rebalance" : "6"
  },
  "defaults" : {
    "action.auto_create_index" : "true",
    "action.destructive_requires_name" : "true",
    "action.search.pre_filter_shard_size.default" : "128",
    "action.search.shard_count.limit" : "9223372036854775807",
    "async_search.index_cleanup_interval" : "1h",
    "bootstrap.ctrlhandler" : "true",
    "bootstrap.memory_lock" : "true",
    "cache.recycler.page.limit.heap" : "10%",
    "cache.recycler.page.type" : "CONCURRENT",
    "cache.recycler.page.weight.bytes" : "1.0",
    "cache.recycler.page.weight.ints" : "1.0",
    "cache.recycler.page.weight.longs" : "1.0",
    "cache.recycler.page.weight.objects" : "0.1",
    "ccr.auto_follow.wait_for_metadata_timeout" : "60s",
    "ccr.indices.recovery.chunk_size" : "1mb",
    "ccr.indices.recovery.internal_action_timeout" : "60s",
    "ccr.indices.recovery.max_bytes_per_sec" : "40mb",
    "ccr.indices.recovery.max_concurrent_file_chunks" : "5",
    "ccr.indices.recovery.recovery_activity_timeout" : "60s",
    "ccr.wait_for_metadata_timeout" : "60s",
    "client.type" : "node",
    "cluster.auto_shrink_voting_configuration" : "true",
    "cluster.blocks.read_only" : "false",
    "cluster.blocks.read_only_allow_delete" : "false",
    "cluster.deprecation_indexing.enabled" : "true",
    "cluster.deprecation_indexing.x_opaque_id_used.enabled" : "true",
    "cluster.election.back_off_time" : "100ms",
    "cluster.election.duration" : "500ms",
    "cluster.election.initial_timeout" : "100ms",
    "cluster.election.max_timeout" : "10s",
    "cluster.election.strategy" : "supports_voting_only",
    "cluster.fault_detection.follower_check.interval" : "1000ms",
    "cluster.fault_detection.follower_check.retry_count" : "3",
    "cluster.fault_detection.follower_check.timeout" : "10000ms",
    "cluster.fault_detection.leader_check.interval" : "1000ms",
    "cluster.fault_detection.leader_check.retry_count" : "3",
    "cluster.fault_detection.leader_check.timeout" : "10000ms",
    "cluster.follower_lag.timeout" : "90000ms",
    "cluster.indices.close.enable" : "true",
    "cluster.indices.tombstones.size" : "500",
    "cluster.info.update.interval" : "30s",
    "cluster.info.update.timeout" : "15s",
    "cluster.initial_master_nodes" : [
      "es_master_1_1",
      "es_master_2_1",
      "es_master_1_2",
      "es_master_2_2",
      "es_master_1_3",
      "es_master_2_3"
    ],
    "cluster.max_shards_per_node.frozen" : "3000",
    "cluster.max_voting_config_exclusions" : "10",
    "cluster.name" : "elk_cluster",
    "cluster.no_master_block" : "write",
    "cluster.nodes.reconnect_interval" : "10s",
    "cluster.persistent_tasks.allocation.enable" : "all",
    "cluster.persistent_tasks.allocation.recheck_interval" : "30s",
    "cluster.publish.info_timeout" : "10000ms",
    "cluster.publish.timeout" : "30000ms",
    "cluster.remote.connections_per_cluster" : "3",
    "cluster.remote.initial_connect_timeout" : "30s",
    "cluster.remote.node.attr" : "",
    "cluster.routing.allocation.awareness.attributes" : [ ],
    "cluster.routing.allocation.balance.index" : "0.55",
    "cluster.routing.allocation.balance.shard" : "0.45",
    "cluster.routing.allocation.balance.threshold" : "1.0",
    "cluster.routing.allocation.disk.reroute_interval" : "60s",
    "cluster.routing.allocation.disk.threshold_enabled" : "true",
    "cluster.routing.allocation.disk.watermark.enable_for_single_data_node" : "true",
    "cluster.routing.allocation.disk.watermark.flood_stage.frozen" : "95%",
    "cluster.routing.allocation.disk.watermark.flood_stage.frozen.max_headroom" : "20GB",
    "cluster.routing.allocation.enforce_default_tier_preference" : "true",
    "cluster.routing.allocation.node_concurrent_incoming_recoveries" : "6",
    "cluster.routing.allocation.node_concurrent_outgoing_recoveries" : "6",
    "cluster.routing.allocation.node_initial_primaries_recoveries" : "4",
    "cluster.routing.allocation.same_shard.host" : "false",
    "cluster.routing.allocation.total_shards_per_node" : "-1",
    "cluster.routing.allocation.type" : "balanced",
    "cluster.routing.use_adaptive_replica_selection" : "true",
    "cluster.service.master_service_starvation_logging_threshold" : "5m",
    "cluster.service.slow_master_task_logging_threshold" : "10s",
    "cluster.service.slow_task_logging_threshold" : "30s",
    "cluster.snapshot.info.max_concurrent_fetches" : "5",
    "cluster_state.document_page_size" : "1mb",
    "deprecation.skip_deprecated_settings" : [ ],
    "discovery.cluster_formation_warning_timeout" : "10000ms",
    "discovery.find_peers_interval" : "1000ms",
    "discovery.initial_state_timeout" : "30s",
    "discovery.probe.connect_timeout" : "30s",
    "discovery.probe.handshake_timeout" : "30s",
    "discovery.request_peers_timeout" : "3000ms",
    "discovery.seed_hosts" : [
      "es_master_1_1",
      "es_master_2_1",
      "es_master_1_2",
      "es_master_2_2",
      "es_master_1_3",
      "es_master_2_3",
      "es_data_ssd_1_1",
      "es_data_ssd_2_1",
      "es_data_ssd_3_1",
      "es_data_ssd_4_1",
      "es_data_ssd_5_1",
      "es_data_hdd_1_1",
      "es_data_hdd_4_1",
      "es_data_hdd_7_1",
      "es_data_ssd_3_2_ingest",
      "es_data_ssd_1_2",
      "es_data_ssd_2_2",
      "es_data_ssd_3_2",
      "es_data_ssd_4_2",
      "es_data_ssd_5_2",
      "es_data_hdd_1_2",
      "es_data_hdd_4_2",
      "es_data_hdd_7_2",
      "es_data_ssd_3_3_ingest",
      "es_data_ssd_1_3",
      "es_data_ssd_2_3",
      "es_data_ssd_3_3",
      "es_data_ssd_4_3",
      "es_data_ssd_5_3",
      "es_data_hdd_1_3",
      "es_data_hdd_4_3",
      "es_data_hdd_7_3"
    ],
    "discovery.seed_providers" : [ ],
    "discovery.seed_resolver.max_concurrent_resolvers" : "10",
    "discovery.seed_resolver.timeout" : "5s",
    "discovery.type" : "multi-node",
    "discovery.unconfigured_bootstrap_timeout" : "3s",
    "enrich.cache_size" : "1000",
    "enrich.cleanup_period" : "15m",
    "enrich.coordinator_proxy.max_concurrent_requests" : "8",
    "enrich.coordinator_proxy.max_lookups_per_request" : "128",
    "enrich.coordinator_proxy.queue_capacity" : "1024",
    "enrich.fetch_size" : "10000",
    "enrich.max_concurrent_policy_executions" : "50",
    "enrich.max_force_merge_attempts" : "3",
    "gateway.expected_data_nodes" : "-1",
    "gateway.recover_after_data_nodes" : "-1",
    "gateway.recover_after_time" : "0ms",
    "gateway.slow_write_logging_threshold" : "10s",
    "gateway.write_dangling_indices_info" : "true",
    "http.bind_host" : [ ],
    "http.client_stats.closed_channels.max_age" : "5m",
    "http.client_stats.closed_channels.max_count" : "10000",
    "http.client_stats.enabled" : "true",
    "http.compression" : "true",
    "http.compression_level" : "3",
    "http.cors.allow-credentials" : "false",
    "http.cors.allow-headers" : "X-Requested-With,Content-Type,Content-Length",
    "http.cors.allow-methods" : "OPTIONS,HEAD,GET,POST,PUT,DELETE",
    "http.cors.allow-origin" : "",
    "http.cors.enabled" : "false",
    "http.cors.max-age" : "1728000",
    "http.detailed_errors.enabled" : "true",
    "http.host" : [ ],
    "http.max_chunk_size" : "8kb",
    "http.max_content_length" : "220mb",
    "http.max_header_size" : "8kb",
    "http.max_initial_line_length" : "4kb",
    "http.max_warning_header_count" : "-1",
    "http.max_warning_header_size" : "-1b",
    "http.netty.max_composite_buffer_components" : "69905",
    "http.netty.receive_predictor_size" : "64kb",
    "http.netty.worker_count" : "0",
    "http.pipelining.max_events" : "10000",
    "http.port" : "9200-9300",
    "http.publish_host" : [ ],
    "http.publish_port" : "-1",
    "http.read_timeout" : "0ms",
    "http.reset_cookies" : "false",
    "http.tcp.keep_alive" : "true",
    "http.tcp.keep_count" : "-1",
    "http.tcp.keep_idle" : "-1",
    "http.tcp.keep_interval" : "-1",
    "http.tcp.no_delay" : "true",
    "http.tcp.receive_buffer_size" : "-1b",
    "http.tcp.reuse_address" : "true",
    "http.tcp.send_buffer_size" : "-1b",
    "http.tracer.exclude" : [ ],
    "http.tracer.include" : [ ],
    "http.type" : "security4",
    "http.type.default" : "netty4",
    "index.codec" : "default",
    "index.recovery.type" : "",
    "index.store.fs.fs_lock" : "native",
    "index.store.preload" : [ ],
    "index.store.snapshot.cache.excluded_file_types" : [ ],
    "index.store.snapshot.uncached_chunk_size" : "-1b",
    "index.store.type" : "",
    "indexing_pressure.memory.limit" : "10%",
    "indices.analysis.hunspell.dictionary.ignore_case" : "false",
    "indices.analysis.hunspell.dictionary.lazy" : "false",
    "indices.breaker.fielddata.limit" : "40%",
    "indices.breaker.fielddata.overhead" : "1.03",
    "indices.breaker.fielddata.type" : "memory",
    "indices.breaker.request.overhead" : "1.0",
    "indices.breaker.request.type" : "memory",
    "indices.breaker.total.use_real_memory" : "true",
    "indices.breaker.type" : "hierarchy",
    "indices.cache.cleanup_interval" : "1m",
    "indices.fielddata.cache.size" : "-1b",
    "indices.id_field_data.enabled" : "false",
    "indices.lifecycle.history_index_enabled" : "true",
    "indices.lifecycle.poll_interval" : "10m",
    "indices.lifecycle.step.master_timeout" : "30s",
    "indices.mapping.dynamic_timeout" : "30s",
    "indices.mapping.max_in_flight_updates" : "10",
    "indices.memory.index_buffer_size" : "10%",
    "indices.memory.interval" : "5s",
    "indices.memory.max_index_buffer_size" : "-1",
    "indices.memory.min_index_buffer_size" : "48mb",
    "indices.memory.shard_inactive_time" : "5m",
    "indices.queries.cache.all_segments" : "false",
    "indices.queries.cache.count" : "10000",
    "indices.queries.cache.size" : "10%",
    "indices.query.bool.max_clause_count" : "4096",
    "indices.query.bool.max_nested_depth" : "20",
    "indices.query.query_string.allowLeadingWildcard" : "true",
    "indices.query.query_string.analyze_wildcard" : "false",
    "indices.recovery.internal_action_long_timeout" : "1800000ms",
    "indices.recovery.internal_action_retry_timeout" : "1m",
    "indices.recovery.internal_action_timeout" : "15m",
    "indices.recovery.max_bytes_per_sec" : "40mb",
    "indices.recovery.max_concurrent_operations" : "1",
    "indices.recovery.max_concurrent_snapshot_file_downloads" : "5",
    "indices.recovery.max_concurrent_snapshot_file_downloads_per_node" : "25",
    "indices.recovery.recovery_activity_timeout" : "1800000ms",
    "indices.recovery.retry_delay_network" : "5s",
    "indices.recovery.retry_delay_state_sync" : "500ms",
    "indices.recovery.use_snapshots" : "true",
    "indices.replication.initial_retry_backoff_bound" : "50ms",
    "indices.replication.retry_timeout" : "60s",
    "indices.requests.cache.expire" : "0ms",
    "indices.requests.cache.size" : "1%",
    "indices.store.delete.shard.timeout" : "30s",
    "ingest.geoip.cache_size" : "1000",
    "ingest.geoip.downloader.enabled" : "false",
    "ingest.geoip.downloader.endpoint" : "https://geoip.elastic.co/v1/database",
    "ingest.geoip.downloader.poll.interval" : "3d",
    "ingest.grok.watchdog.interval" : "1s",
    "ingest.grok.watchdog.max_execution_time" : "1s",
    "ingest.user_agent.cache_size" : "1000",
    "logger.level" : "INFO",
    "monitor.fs.health.enabled" : "true",
    "monitor.fs.health.refresh_interval" : "120s",
    "monitor.fs.health.slow_path_logging_threshold" : "5s",
    "monitor.fs.refresh_interval" : "1s",
    "monitor.jvm.gc.enabled" : "true",
    "monitor.jvm.gc.overhead.debug" : "10",
    "monitor.jvm.gc.overhead.info" : "25",
    "monitor.jvm.gc.overhead.warn" : "50",
    "monitor.jvm.gc.refresh_interval" : "1s",
    "monitor.jvm.refresh_interval" : "1s",
    "monitor.os.refresh_interval" : "1s",
    "monitor.process.refresh_interval" : "1s",
    "network.bind_host" : [
      "0.0.0.0"
    ],
    "network.breaker.inflight_requests.limit" : "100%",
    "network.breaker.inflight_requests.overhead" : "2.0",
    "network.host" : [
      "0.0.0.0"
    ],
    "network.publish_host" : [
      "0.0.0.0"
    ],
    "network.server" : "true",
    "network.tcp.keep_alive" : "true",
    "network.tcp.keep_count" : "-1",
    "network.tcp.keep_idle" : "-1",
    "network.tcp.keep_interval" : "-1",
    "network.tcp.no_delay" : "true",
    "network.tcp.receive_buffer_size" : "-1b",
    "network.tcp.reuse_address" : "true",
    "network.tcp.send_buffer_size" : "-1b",
    "node.attr.rack_id" : "rack_one",
    "node.attr.xpack.installed" : "true",
    "node.bandwidth.recovery.disk.read" : "-1",
    "node.bandwidth.recovery.disk.write" : "-1",
    "node.bandwidth.recovery.factor.read" : "0.4",
    "node.bandwidth.recovery.factor.write" : "0.4",
    "node.bandwidth.recovery.network" : "-1",
    "node.bandwidth.recovery.operator.factor" : "0.4",
    "node.bandwidth.recovery.operator.factor.max_overcommit" : "100.0",
    "node.bandwidth.recovery.operator.factor.read" : "0.4",
    "node.bandwidth.recovery.operator.factor.write" : "0.4",
    "node.enable_lucene_segment_infos_trace" : "false",
    "node.external_id" : "",
    "node.id.seed" : "0",
    "node.name" : "es_data_ssd_3_1_ingest",
    "node.pidfile" : "",
    "node.portsfile" : "false",
    "node.processors" : "64",
    "node.roles" : [
      "ingest"
    ],
    "node.store.allow_mmap" : "true",
    "path.data" : [ ],
    "path.home" : "/usr/share/elasticsearch",
    "path.logs" : "/usr/share/elasticsearch/logs",
    "path.repo" : [ ],
    "path.shared_data" : "",
    "plugin.mandatory" : [ ],
    "reindex.remote.whitelist" : [ ],
    "repositories.fs.chunk_size" : "9223372036854775807b",
    "repositories.fs.location" : "",
    "repositories.url.allowed_urls" : [ ],
    "repositories.url.supported_protocols" : [
      "http",
      "https",
      "ftp",
      "file",
      "jar"
    ],
    "repositories.url.url" : "http:",
    "resource.reload.enabled" : "true",
    "resource.reload.interval.high" : "5s",
    "resource.reload.interval.low" : "60s",
    "resource.reload.interval.medium" : "30s",
    "rest.action.multi.allow_explicit_index" : "true",
    "runtime_fields.grok.watchdog.interval" : "1s",
    "runtime_fields.grok.watchdog.max_execution_time" : "1s",
    "script.allowed_contexts" : [ ],
    "script.allowed_types" : [ ],
    "script.cache.expire" : "0ms",
    "script.cache.max_size" : "3000",
    "script.disable_max_compilations_rate" : "false",
    "script.max_compilations_rate" : "150/5m",
    "script.max_size_in_bytes" : "65535",
    "script.painless.regex.enabled" : "limited",
    "script.painless.regex.limit-factor" : "6",
    "search.aggs.rewrite_to_filter_by_filter" : "true",
    "search.allow_expensive_queries" : "true",
    "search.check_ccs_compatibility" : "false",
    "search.default_allow_partial_results" : "true",
    "search.default_keep_alive" : "5m",
    "search.default_search_timeout" : "-1",
    "search.highlight.term_vector_multi_value" : "true",
    "search.keep_alive_interval" : "1m",
    "search.low_level_cancellation" : "true",
    "search.max_async_search_response_size" : "10mb",
    "search.max_buckets" : "65536",
    "search.max_keep_alive" : "24h",
    "search.max_open_scroll_context" : "500",
    "searchable_snapshots.blob_cache.periodic_cleanup.batch_size" : "100",
    "searchable_snapshots.blob_cache.periodic_cleanup.interval" : "1h",
    "searchable_snapshots.blob_cache.periodic_cleanup.pit_keep_alive" : "10m",
    "searchable_snapshots.blob_cache.periodic_cleanup.retention_period" : "1h",
    "security.manager.filter_bad_defaults" : "true",
    "slm.history_index_enabled" : "true",
    "slm.minimum_interval" : "15m",
    "slm.retention_duration" : "1h",
    "slm.retention_schedule" : "0 30 1 * * ?",
    "snapshot.max_concurrent_operations" : "1000",
    "snapshot.refresh_repo_uuid_on_restore" : "true",
    "stack.templates.enabled" : "true",
    "thread_pool.analyze.queue_size" : "16",
    "thread_pool.analyze.size" : "1",
    "thread_pool.auto_complete.queue_size" : "100",
    "thread_pool.auto_complete.size" : "16",
    "thread_pool.azure_event_loop.core" : "0",
    "thread_pool.azure_event_loop.keep_alive" : "30s",
    "thread_pool.azure_event_loop.max" : "1",
    "thread_pool.estimated_time_interval" : "200ms",
    "thread_pool.estimated_time_interval.warn_threshold" : "5s",
    "thread_pool.fetch_shard_started.core" : "1",
    "thread_pool.fetch_shard_started.keep_alive" : "5m",
    "thread_pool.fetch_shard_started.max" : "128",
    "thread_pool.fetch_shard_store.core" : "1",
    "thread_pool.fetch_shard_store.keep_alive" : "5m",
    "thread_pool.fetch_shard_store.max" : "128",
    "thread_pool.flush.core" : "1",
    "thread_pool.flush.keep_alive" : "5m",
    "thread_pool.flush.max" : "5",
    "thread_pool.force_merge.queue_size" : "-1",
    "thread_pool.force_merge.size" : "1",
    "thread_pool.generic.core" : "4",
    "thread_pool.generic.keep_alive" : "30s",
    "thread_pool.generic.max" : "256",
    "thread_pool.get.queue_size" : "1000",
    "thread_pool.get.size" : "64",
    "thread_pool.management.core" : "1",
    "thread_pool.management.keep_alive" : "5m",
    "thread_pool.management.max" : "5",
    "thread_pool.refresh.core" : "1",
    "thread_pool.refresh.keep_alive" : "5m",
    "thread_pool.refresh.max" : "10",
    "thread_pool.repository_azure.core" : "0",
    "thread_pool.repository_azure.keep_alive" : "30s",
    "thread_pool.repository_azure.max" : "5",
    "thread_pool.scheduler.warn_threshold" : "5s",
    "thread_pool.search.queue_size" : "1000",
    "thread_pool.search.size" : "97",
    "thread_pool.search_coordination.queue_size" : "1000",
    "thread_pool.search_coordination.size" : "5",
    "thread_pool.search_throttled.queue_size" : "100",
    "thread_pool.search_throttled.size" : "1",
    "thread_pool.snapshot.core" : "1",
    "thread_pool.snapshot.keep_alive" : "5m",
    "thread_pool.snapshot.max" : "5",
    "thread_pool.snapshot_meta.core" : "1",
    "thread_pool.snapshot_meta.keep_alive" : "30s",
    "thread_pool.snapshot_meta.max" : "50",
    "thread_pool.system_critical_read.queue_size" : "2000",
    "thread_pool.system_critical_read.size" : "5",
    "thread_pool.system_critical_write.queue_size" : "1500",
    "thread_pool.system_critical_write.size" : "5",
    "thread_pool.system_read.queue_size" : "2000",
    "thread_pool.system_read.size" : "5",
    "thread_pool.system_write.queue_size" : "1000",
    "thread_pool.system_write.size" : "5",
    "thread_pool.vectortile.queue_size" : "-1",
    "thread_pool.vectortile.size" : "1",
    "thread_pool.warmer.core" : "1",
    "thread_pool.warmer.keep_alive" : "5m",
    "thread_pool.warmer.max" : "5",
    "thread_pool.write.queue_size" : "10000",
    "thread_pool.write.size" : "64",
    "transport.bind_host" : [ ],
    "transport.compress" : "INDEXING_DATA",
    "transport.compression_scheme" : "LZ4",
    "transport.connect_timeout" : "30s",
    "transport.connections_per_node.bulk" : "3",
    "transport.connections_per_node.ping" : "1",
    "transport.connections_per_node.recovery" : "2",
    "transport.connections_per_node.reg" : "6",
    "transport.connections_per_node.state" : "1",
    "transport.host" : [ ],
    "transport.netty.boss_count" : "1",
    "transport.netty.receive_predictor_max" : "64kb",
    "transport.netty.receive_predictor_min" : "64kb",
    "transport.netty.receive_predictor_size" : "64kb",
    "transport.netty.worker_count" : "64",
    "transport.ping_schedule" : "-1",
    "transport.port" : "9300-9400",
    "transport.publish_host" : [ ],
    "transport.publish_port" : "-1",
    "transport.rst_on_close" : "false",
    "transport.slow_operation_logging_threshold" : "5s",
    "transport.tcp.keep_alive" : "true",
    "transport.tcp.keep_count" : "-1",
    "transport.tcp.keep_idle" : "-1",
    "transport.tcp.keep_interval" : "-1",
    "transport.tcp.no_delay" : "true",
    "transport.tcp.receive_buffer_size" : "-1b",
    "transport.tcp.reuse_address" : "true",
    "transport.tcp.send_buffer_size" : "-1b",
    "transport.tracer.exclude" : [
      "internal:coordination/fault_detection/*"
    ],
    "transport.tracer.include" : [ ],
    "transport.type" : "security4",
    "transport.type.default" : "netty4",
    "xpack.autoscaling.memory.monitor.timeout" : "15s",
    "xpack.ccr.ccr_thread_pool.queue_size" : "100",
    "xpack.ccr.ccr_thread_pool.size" : "32",
    "xpack.ccr.enabled" : "true",
    "xpack.eql.enabled" : "true",
    "xpack.graph.enabled" : "true",
    "xpack.http.connection_pool_ttl" : "-1",
    "xpack.http.default_connection_timeout" : "10s",
    "xpack.http.default_read_timeout" : "10s",
    "xpack.http.max_response_size" : "10mb",
    "xpack.http.proxy.host" : "",
    "xpack.http.proxy.port" : "0",
    "xpack.http.proxy.scheme" : "",
    "xpack.http.tcp.keep_alive" : "true",
    "xpack.http.whitelist" : [
      "*"
    ],
    "xpack.idp.allowed_nameid_formats" : [
      "urn:oasis:names:tc:SAML:2.0:nameid-format:transient"
    ],
    "xpack.idp.contact.email" : "",
    "xpack.idp.contact.given_name" : "",
    "xpack.idp.contact.surname" : "",
    "xpack.idp.defaults.authn_expiry" : "5m",
    "xpack.idp.defaults.nameid_format" : "urn:oasis:names:tc:SAML:2.0:nameid-format:transient",
    "xpack.idp.enabled" : "false",
    "xpack.idp.entity_id" : "",
    "xpack.idp.metadata.signing.keystore.alias" : "",
    "xpack.idp.organization.display_name" : "",
    "xpack.idp.organization.name" : "",
    "xpack.idp.organization.url" : "http:",
    "xpack.idp.privileges.application" : "",
    "xpack.idp.privileges.cache.size" : "100",
    "xpack.idp.privileges.cache.ttl" : "90m",
    "xpack.idp.signing.keystore.alias" : "",
    "xpack.idp.slo_endpoint.post" : "https:",
    "xpack.idp.slo_endpoint.redirect" : "https:",
    "xpack.idp.sp.cache.size" : "1000",
    "xpack.idp.sp.cache.ttl" : "60m",
    "xpack.idp.sp.wildcard.path" : "wildcard_services.json",
    "xpack.idp.sso_endpoint.post" : "https:",
    "xpack.idp.sso_endpoint.redirect" : "https:",
    "xpack.license.self_generated.type" : "basic",
    "xpack.license.upload.types" : [
      "standard",
      "gold",
      "platinum",
      "enterprise",
      "trial"
    ],
    "xpack.ml.autodetect_process" : "true",
    "xpack.ml.delayed_data_check_freq" : "15m",
    "xpack.ml.enable_config_migration" : "true",
    "xpack.ml.enabled" : "false",
    "xpack.ml.inference_model.cache_size" : "40%",
    "xpack.ml.inference_model.time_to_live" : "5m",
    "xpack.ml.max_anomaly_records" : "500",
    "xpack.ml.max_inference_processors" : "50",
    "xpack.ml.max_lazy_ml_nodes" : "0",
    "xpack.ml.max_machine_memory_percent" : "30",
    "xpack.ml.max_ml_node_size" : "0b",
    "xpack.ml.max_model_memory_limit" : "0b",
    "xpack.ml.max_open_jobs" : "512",
    "xpack.ml.min_disk_space_off_heap" : "5gb",
    "xpack.ml.nightly_maintenance_requests_per_second" : "-1.0",
    "xpack.ml.node_concurrent_job_allocations" : "2",
    "xpack.ml.persist_results_max_retries" : "20",
    "xpack.ml.process_connect_timeout" : "10s",
    "xpack.ml.use_auto_machine_memory_percent" : "false",
    "xpack.monitoring.collection.ccr.stats.timeout" : "10s",
    "xpack.monitoring.collection.cluster.stats.timeout" : "10s",
    "xpack.monitoring.collection.enrich.stats.timeout" : "10s",
    "xpack.monitoring.collection.index.recovery.active_only" : "false",
    "xpack.monitoring.collection.index.recovery.timeout" : "10s",
    "xpack.monitoring.collection.index.stats.timeout" : "10s",
    "xpack.monitoring.collection.indices" : [ ],
    "xpack.monitoring.collection.interval" : "10s",
    "xpack.monitoring.collection.ml.job.stats.timeout" : "10s",
    "xpack.monitoring.collection.node.stats.timeout" : "10s",
    "xpack.monitoring.history.duration" : "168h",
    "xpack.monitoring.migration.decommission_alerts" : "false",
    "xpack.monitoring.templates.enabled" : "true",
    "xpack.notification.email.default_account" : "",
    "xpack.notification.email.html.sanitization.allow" : [
      "body",
      "head",
      "_tables",
      "_links",
      "_blocks",
      "_formatting",
      "img:embedded"
    ],
    "xpack.notification.email.html.sanitization.disallow" : [ ],
    "xpack.notification.email.html.sanitization.enabled" : "true",
    "xpack.notification.jira.default_account" : "",
    "xpack.notification.pagerduty.default_account" : "",
    "xpack.notification.reporting.interval" : "15s",
    "xpack.notification.reporting.retries" : "40",
    "xpack.notification.reporting.warning.enabled" : "true",
    "xpack.notification.slack.default_account" : "",
    "xpack.rollup.task_thread_pool.queue_size" : "-1",
    "xpack.rollup.task_thread_pool.size" : "1",
    "xpack.searchable.snapshot.allocate_on_rolling_restart" : "false",
    "xpack.searchable.snapshot.cache.range_size" : "32mb",
    "xpack.searchable.snapshot.cache.recovery_range_size" : "128kb",
    "xpack.searchable.snapshot.cache.sync.interval" : "60s",
    "xpack.searchable.snapshot.cache.sync.max_files" : "10000",
    "xpack.searchable.snapshot.cache.sync.shutdown_timeout" : "10s",
    "xpack.searchable.snapshot.shared_cache.decay.interval" : "60s",
    "xpack.searchable.snapshot.shared_cache.max_freq" : "100",
    "xpack.searchable.snapshot.shared_cache.min_time_delta" : "60s",
    "xpack.searchable.snapshot.shared_cache.range_size" : "16mb",
    "xpack.searchable.snapshot.shared_cache.recovery_range_size" : "128kb",
    "xpack.searchable.snapshot.shared_cache.region_size" : "16mb",
    "xpack.searchable.snapshot.shared_cache.size" : "0",
    "xpack.searchable.snapshot.shared_cache.size.max_headroom" : "-1",
    "xpack.searchable_snapshots.cache_fetch_async_thread_pool.core" : "0",
    "xpack.searchable_snapshots.cache_fetch_async_thread_pool.keep_alive" : "30s",
    "xpack.searchable_snapshots.cache_fetch_async_thread_pool.max" : "50",
    "xpack.searchable_snapshots.cache_prewarming_thread_pool.core" : "0",
    "xpack.searchable_snapshots.cache_prewarming_thread_pool.keep_alive" : "30s",
    "xpack.searchable_snapshots.cache_prewarming_thread_pool.max" : "16",
    "xpack.security.audit.enabled" : "false",
    "xpack.security.audit.logfile.emit_cluster_name" : "false",
    "xpack.security.audit.logfile.emit_cluster_uuid" : "true",
    "xpack.security.audit.logfile.emit_node_host_address" : "false",
    "xpack.security.audit.logfile.emit_node_host_name" : "false",
    "xpack.security.audit.logfile.emit_node_id" : "true",
    "xpack.security.audit.logfile.emit_node_name" : "false",
    "xpack.security.audit.logfile.events.emit_request_body" : "false",
    "xpack.security.audit.logfile.events.exclude" : [ ],
    "xpack.security.audit.logfile.events.include" : [
      "ACCESS_DENIED",
      "ACCESS_GRANTED",
      "ANONYMOUS_ACCESS_DENIED",
      "AUTHENTICATION_FAILED",
      "CONNECTION_DENIED",
      "TAMPERED_REQUEST",
      "RUN_AS_DENIED",
      "RUN_AS_GRANTED",
      "SECURITY_CONFIG_CHANGE"
    ],
    "xpack.security.authc.anonymous.authz_exception" : "true",
    "xpack.security.authc.anonymous.roles" : [ ],
    "xpack.security.authc.anonymous.username" : "_anonymous",
    "xpack.security.authc.api_key.cache.hash_algo" : "ssha256",
    "xpack.security.authc.api_key.cache.max_keys" : "25000",
    "xpack.security.authc.api_key.cache.ttl" : "24h",
    "xpack.security.authc.api_key.delete.interval" : "24h",
    "xpack.security.authc.api_key.delete.timeout" : "-1",
    "xpack.security.authc.api_key.doc_cache.ttl" : "5m",
    "xpack.security.authc.api_key.enabled" : "true",
    "xpack.security.authc.api_key.hashing.algorithm" : "pbkdf2",
    "xpack.security.authc.password_hashing.algorithm" : "BCRYPT",
    "xpack.security.authc.reserved_realm.enabled" : "true",
    "xpack.security.authc.run_as.enabled" : "true",
    "xpack.security.authc.service_token.cache.hash_algo" : "ssha256",
    "xpack.security.authc.service_token.cache.max_tokens" : "100000",
    "xpack.security.authc.service_token.cache.ttl" : "20m",
    "xpack.security.authc.success_cache.enabled" : "true",
    "xpack.security.authc.success_cache.expire_after_access" : "1h",
    "xpack.security.authc.success_cache.size" : "10000",
    "xpack.security.authc.token.delete.interval" : "30m",
    "xpack.security.authc.token.delete.timeout" : "-1",
    "xpack.security.authc.token.enabled" : "false",
    "xpack.security.authc.token.thread_pool.queue_size" : "1000",
    "xpack.security.authc.token.thread_pool.size" : "1",
    "xpack.security.authc.token.timeout" : "20m",
    "xpack.security.authz.store.privileges.cache.max_size" : "10000",
    "xpack.security.authz.store.privileges.cache.ttl" : "24h",
    "xpack.security.authz.store.roles.cache.max_size" : "10000",
    "xpack.security.authz.store.roles.field_permissions.cache.max_size_in_bytes" : "104857600",
    "xpack.security.authz.store.roles.negative_lookup_cache.max_size" : "10000",
    "xpack.security.authz.timer.indices.enabled" : "false",
    "xpack.security.authz.timer.indices.threshold.debug" : "20ms",
    "xpack.security.authz.timer.indices.threshold.info" : "100ms",
    "xpack.security.authz.timer.indices.threshold.warn" : "200ms",
    "xpack.security.autoconfiguration.enabled" : "true",
    "xpack.security.automata.cache.enabled" : "true",
    "xpack.security.automata.cache.size" : "10000",
    "xpack.security.automata.cache.ttl" : "48h",
    "xpack.security.automata.max_determinized_states" : "100000",
    "xpack.security.crypto.thread_pool.queue_size" : "1000",
    "xpack.security.crypto.thread_pool.size" : "32",
    "xpack.security.dls.bitset.cache.size" : "10%",
    "xpack.security.dls.bitset.cache.ttl" : "2h",
    "xpack.security.dls_fls.enabled" : "true",
    "xpack.security.enabled" : "true",
    "xpack.security.encryption.algorithm" : "AES/CTR/NoPadding",
    "xpack.security.encryption_key.algorithm" : "AES",
    "xpack.security.encryption_key.length" : "128",
    "xpack.security.enrollment.enabled" : "false",
    "xpack.security.filter.always_allow_bound_address" : "true",
    "xpack.security.fips_mode.enabled" : "false",
    "xpack.security.http.filter.allow" : [ ],
    "xpack.security.http.filter.deny" : [ ],
    "xpack.security.http.filter.enabled" : "true",
    "xpack.security.http.ssl.enabled" : "false",
    "xpack.security.operator_privileges.enabled" : "false",
    "xpack.security.ssl.diagnose.trust" : "true",
    "xpack.security.transport.filter.allow" : [ ],
    "xpack.security.transport.filter.deny" : [ ],
    "xpack.security.transport.filter.enabled" : "true",
    "xpack.security.transport.ssl.enabled" : "true",
    "xpack.security.user" : null,
    "xpack.transform.num_transform_failure_retries" : "10",
    "xpack.watcher.actions.bulk.default_timeout" : "",
    "xpack.watcher.actions.index.default_timeout" : "",
    "xpack.watcher.bulk.actions" : "1",
    "xpack.watcher.bulk.concurrent_requests" : "0",
    "xpack.watcher.bulk.flush_interval" : "1s",
    "xpack.watcher.bulk.size" : "1mb",
    "xpack.watcher.enabled" : "true",
    "xpack.watcher.encrypt_sensitive_data" : "false",
    "xpack.watcher.execution.default_throttle_period" : "5s",
    "xpack.watcher.execution.scroll.size" : "0",
    "xpack.watcher.execution.scroll.timeout" : "",
    "xpack.watcher.index.rest.direct_access" : "",
    "xpack.watcher.input.search.default_timeout" : "",
    "xpack.watcher.internal.ops.bulk.default_timeout" : "",
    "xpack.watcher.internal.ops.index.default_timeout" : "",
    "xpack.watcher.internal.ops.search.default_timeout" : "",
    "xpack.watcher.stop.timeout" : "30s",
    "xpack.watcher.thread_pool.queue_size" : "1000",
    "xpack.watcher.thread_pool.size" : "1",
    "xpack.watcher.transform.search.default_timeout" : "",
    "xpack.watcher.trigger.schedule.ticker.tick_interval" : "500ms",
    "xpack.watcher.use_ilm_index_management" : "true",
    "xpack.watcher.watch.scroll.size" : "0"
  }
}
5

What do You think does it optimal settings?

6

The docs for this setting say:

Increasing this setting may cause the cluster to use additional resources moving shards between nodes, so we generally do not recommend adjusting this setting from its default of 2 .

Instead, just wait patiently for the cluster to finish rebalancing.