Hi,
I'm using 3 nodes cluster and monthly indices with 6 primary and 1 replica shards.
es-node-1 Hot node
es-node-2 Warm node
es-node-3 Cold node
Using the ILM policy with no rollover and the monthly indices stuck at allocation action onto Warm node. The indices waiting about 4 days to allocate.
I tested this lifecycle way a month ago and it worked fine. But now the lifecycle is not working as expected. How can I resolve this issue?
ILM explain:
Blockquote
{
"indices" : {
"document-2019.10" : {
"index" : "document-2019.10",
"managed" : true,
"policy" : "grid-ilm-policy",
"lifecycle_date_millis" : 1571048211487,
"phase" : "warm",
"phase_time_millis" : 1573813256180,
"action" : "allocate",
"action_time_millis" : 1573813856327,
"step" : "check-allocation",
"step_time_millis" : 1573813856438,
"step_info" : {
"message" : "Waiting for [6] shards to be allocated to nodes matching the given filters",
"shards_left_to_allocate" : 6,
"all_shards_active" : true,
"actual_replicas" : 1
},
"phase_execution" : {
"policy" : "grid-ilm-policy",
"phase_definition" : {
"min_age" : "32d",
"actions" : {
"allocate" : {
"include" : { },
"exclude" : { },
"require" : {
"box_type" : "warm"
}
},
"set_priority" : {
"priority" : 50
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"version" : 9,
"modified_date_in_millis" : 1571038556385
}
}
}
}
Index settings:
Blockquote
{
"document-2019.10" : {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "grid-ilm-policy"
},
"routing" : {
"allocation" : {
"require" : {
"box_type" : "warm"
}
}
},
"number_of_shards" : "6",
"provided_name" : "document-2019.10",
"creation_date" : "1571048211487",
"priority" : "50",
"number_of_replicas" : "1",
"uuid" : "9YQB0XJ5SU2aT_L1ZLIWuw",
"version" : {
"created" : "7030299"
}
}
}
}
}