Shards not being allocated according to "box_type" setting in designated nodes of same type

Elastic Stack Elasticsearch
1

I have an index of around 4.1TB~ total size in a cluster of 12 nodes (9 DATA & INGEST, 3 DATA & MASTER) and all of them are set to "hot" category - we don't currently have any "warm" node in this cluster.

The problem is I have 10 replica shards in state UNASSIGNED - I added in the first comment of this post the output from Explain API command regarding this (topic doesn't allow me to enter too many chars):

Every shard is around 300GB~ (above the recommended threshold of 50GB) and initially there were disk space issues in many nodes but then I performed clean up of garbage files there. After this, the explain output stopped complaining about disk watermark hit but keeps complaining about the box_type setting.

Appreciate any help on this.

Regards,
Vinicio

2

{
"index": "puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest",
"shard": 6,
"primary": false,
"current_state": "unassigned",
"unassigned_info": {
"reason": "INDEX_REOPENED",
"at": "2019-08-29T22:58:11.191Z",
"last_allocation_status": "no_attempt"
},
"can_allocate": "no",
"allocate_explanation": "cannot allocate because allocation is not permitted to any of the nodes",
"node_allocation_decisions": [
{
"node_id": "DSE_zvuCQ2mbFjzI1559lA",
"node_name": "data1-iil-002_data",
"transport_address": "10.184.95.2:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "L1f5FmIoQCa4Q-6q26W83w",
"node_name": "data1-iil-004_data",
"transport_address": "10.184.95.4:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "MOHZnSRpQzSr0qI4Ki4aZQ",
"node_name": "data1-iil-008_data",
"transport_address": "10.184.95.61:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "NdV2wnCaTKWjNr1Skdss1g",
"node_name": "data1-iil-001_data",
"transport_address": "10.184.95.1:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "UxYWqOEgRKqPchErIvHCvg",
"node_name": "data1-iil-003_data",
"transport_address": "10.184.95.3:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "Xa-4B4RKRxuYN0gTIZVWoA",
"node_name": "data1-iil-006_data",
"transport_address": "10.184.95.59:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "bx9k4EVnRIWhadtwz7VwJw",
"node_name": "data1-iil-010_data",
"transport_address": "10.184.95.63:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "lCdtln5LSF-7_VQpvFRjiA",
"node_name": "data1-iil-012_data",
"transport_address": "10.184.95.64:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "xzdfB0NNTU61pc5xTHkZlw",
"node_name": "data1-iil-009_data",
"transport_address": "10.184.95.62:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
}
]
},
{
"node_id": "yFOxEiLaQF2DAe5A9XOY6g",
"node_name": "data1-iil-005_data",
"transport_address": "10.184.95.5:9300",
"node_attributes": {
"ibi_site": "iil",
"box_type": "hot"
},
"node_decision": "no",
"deciders": [
{
"decider": "filter",
"decision": "NO",
"explanation": """node does not match index setting [index.routing.allocation.require] filters [box_type:"hot",_name:"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest"]"""
},
{
"decider": "same_shard",
"decision": "NO",
"explanation": "the shard cannot be allocated to the same node on which a copy of the shard already exists [[puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest][6], node[yFOxEiLaQF2DAe5A9XOY6g], [P], s[STARTED], a[id=jSUFYBZQT0eddLe3kotinQ]]"
}
]
}
]
}

3

**Index settings are: (excluding the schema mapping as it's too long too many fields) **

{
"puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest": {
"aliases": {
"puma.compilation.pipeline": {}
},
"mappings": {
"doc": {
"dynamic": "true",
"_all": {
"enabled": false
},
"dynamic_templates": [
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"index": true,
"normalizer": "keyword_lowercase",
"type": "keyword"
}
}
}
]
},
"default": {
"dynamic_templates": [
{
"strings": {
"match_mapping_type": "string",
"mapping": {
"index": true,
"normalizer": "keyword_lowercase",
"type": "keyword"
}
}
}
]
}
},
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"ibi_site": "iil,pdx,png"
},
"require": {
"_name": "puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest",
"box_type": "hot"
}
}
},
"refresh_interval": "30s",
"number_of_shards": "10",
"translog": {
"durability": "async"
},
"auto_expand_replicas": "1-1",
"blocks": {
"write": "false"
},
"provided_name": "puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest",
"creation_date": "1519401031998",
"analysis": {
"normalizer": {
"keyword_lowercase": {
"filter": [
"lowercase"
],
"type": "custom"
}
}
},
"number_of_replicas": "1",
"uuid": "69mUudmtSW2j2qWv8Dj2jQ",
"version": {
"created": "5040099",
"upgraded": "5060299"
}
}
}
}
}

4

Hi @viniciof,

it looks like you have two require criteria for each index: one on box_type and one on _name. Notice that using _name you specify the node name, not the index name. I think you need to remove that part from the require filter?

1 Like
5

Thank you so much!! It works now by assigning that field to NULL :slight_smile:

PUT /puma.compilation.pipeline.96f19f5b-bc84-4d4b-8694-b80a293e78e4-latest/_settings
{
"settings": {
"index.routing.allocation.require._name": null
}
}

Found more details on this PRD here Index-level shard allocation filtering | Elasticsearch Guide [8.11] | Elastic for anyone else having similar issues

Regards,

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.