GET _cat/shards
syslog-2019.07.26-000009 0 p STARTED 551317 81mb 172.25.4.176 warm-logging-elk-3
syslog-2019.07.26-000009 0 r STARTED 551317 81.9mb 172.25.4.175 hot-logging-elk-2
GET syslog-2019.07.26*/_settings
"syslog-2019.07.26-000009" : {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "logstash-policy",
"rollover_alias" : "syslog",
"indexing_complete" : "true"
},
"routing" : {
"allocation" : {
"require" : {
"data" : "warm"
}
}
},
"refresh_interval" : "5s",
"number_of_shards" : "1",
"provided_name" : "<syslog-{now/d}-000009>",
"creation_date" : "1564136984891",
"priority" : "100",
"number_of_replicas" : "1",
"version" : {
"created" : "7020099"
}
}
}
}
GET _cluster/state
{
"_nodes": {
"total": 2,
"successful": 2,
"failed": 0
},
"cluster_name": "elk",
"nodes": {
"4cYRknWaRSiff32Y-A29ng": {
"name": "warm-logging-elk-3",
"version": "7.2.0",
"build_flavor": "default",
"build_type": "deb",
"build_hash": "508c38a",
"roles": [
"data"
],
"attributes": {
"ml.machine_memory": "21048377344",
"ml.max_open_jobs": "20",
"xpack.installed": "true",
"data": "warm"
}
},
"lCBklgj7RCat6JP1i3V3tg": {
"name": "hot-logging-elk-2",
"version": "7.2.0",
"build_flavor": "default",
"build_type": "deb",
"build_hash": "508c38a",
"roles": [
"data",
"ingest",
"master"
],
"attributes": {
"ml.machine_memory": "15763595264",
"xpack.installed": "true",
"data": "hot",
"ml.max_open_jobs": "20"
}
}
}
}
- all nodes have settings that contain
"settings.cluster" : {
"name" : "elk",
"routing" : {
"allocation" : {
"allow_rebalance" : "always"
}
}
}
GET /_cluster/allocation/explain
{
"index" : "syslog-2019.07.26-000009",
"shard" : 0,
"primary" : false,
"current_state" : "started",
"current_node" : {
"id" : "lCBWDgj7RCat6JP1i3V3tg",
"name" : "hot-logging-elk",
"transport_address" : "172.25.4.175:9300",
"attributes" : {
"ml.machine_memory" : "15763595264",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"data" : "hot"
}
},
"can_remain_on_current_node" : "no",
"can_remain_decisions" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not match index setting [index.routing.allocation.require] filters [data:"warm"]"""
}
],
"can_move_to_other_node" : "no",
"move_explanation" : "cannot move shard to another node, even though it is not allowed to remain on its current node",
"node_allocation_decisions" : [
{
"node_id" : "4cYRknWMRSiff32Y-A29ng",
"node_name" : "warm-logging-elk-3",
"transport_address" : "172.25.4.176:9300",
"node_attributes" : {
"ml.machine_memory" : "21048377344",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"data" : "warm"
},
"node_decision" : "no",
"weight_ranking" : 1,
"deciders" : [
{
"decider" : "same_shard",
"decision" : "NO",
"explanation" : "the shard cannot be allocated to the same node on which a copy of the shard already exists [[syslog-2019.07.26-000009][0], node[4cYRknWMRSiff32Y-A29ng], [P], s[STARTED], a[id=2vTwS3Z1TGyXbjqfriKKNA]]"
}
]
}
]
}
Executing
PUT syslog*/_settings
{
"index" : {
"number_of_replicas" : 0
}
}
PUT syslog*/_settings
{
"index" : {
"number_of_replicas" : 1
}
}
forces replica to be unassigned resulting in
syslog-2019.07.26-000009 0 p STARTED 551317 81mb 172.25.4.176 warm-logging-elk-3
syslog-2019.07.26-000009 0 r UNASSIGNED
restarting hot node seems to have the same effect.
Based on the output from allocation/explain it seems that elasticsearch decides to keep replica there on purpose even if cluster.routing.allocation.allow_rebalance: always
should allow to perform rebalance that results in cluster unhealthy state