ILM with existing indices

[lchan]

I am following Manage existing indices | Elasticsearch Guide [8.2] | Elastic and I am trying to apply all existing indices (that are in hot node) with policy "existing-indices" so it can go through ilm but it the step is stuck with lifecycle action [migrate] waiting for [1] shards to be moved to the [data_cold] tier (tier migration preference configuration is [data_cold,data_warm,data_hot])",

Why is it waitinig for to move to [data_cold]?

GET filebeat-7.11.1-2021.06.20-000186/_settings

{
  "filebeat-7.11.1-2021.06.20-000186" : {
    "settings" : {
      "index" : {
        "mapping" : {
          "total_fields" : {
            "limit" : "10000"
          }
        },
        "refresh_interval" : "5s",
        "blocks" : {
          "read_only_allow_delete" : "false",
          "write" : "true"
        },
        "provided_name" : "<filebeat-7.11.1-{now/d}-000186>",
        "query" : {
....omitted
            "zeek.notice.identifier",
            "fields.*"
          ]
        },
        "creation_date" : "1624180391747",
        "priority" : "100",
        "number_of_replicas" : "0",
        "uuid" : "mBhYk9dXTJy9rYBGOjRdFg",
        "version" : {
          "created" : "7130199"
        },
        "lifecycle" : {
          "name" : "existing-indices",
          "rollover_alias" : "filebeat-7.11.1",
          "indexing_complete" : "true"
        },
        "routing" : {
          "allocation" : {
            "require" : {
              "data" : "warm"
            }
          }
        },
        "number_of_shards" : "1",
        "max_docvalue_fields_search" : "200"
      }
    }
  }
}

GET /_ilm/policy/existing-indices

{
  "existing-indices" : {
    "version" : 6,
    "modified_date" : "2022-05-20T19:44:32.068Z",
    "policy" : {
      "phases" : {
        "warm" : {
          "min_age" : "1d",
          "actions" : {
            "allocate" : {
              "include" : { },
              "exclude" : { },
              "require" : {
                "data" : "warm"
              }
            },
            "forcemerge" : {
              "max_num_segments" : 1
            }
          }
        },
        "cold" : {
          "min_age" : "30d",
          "actions" : {
            "set_priority" : {
              "priority" : 0
            }
          }
        },
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "set_priority" : {
              "priority" : 100
            }
          }
        },
        "delete" : {
          "min_age" : "365d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  }
}


GET /filebeat-7.11.1-2021.06.20-000186/_ilm/explain

{
  "indices" : {
    "filebeat-7.11.1-2021.06.20-000186" : {
      "index" : "filebeat-7.11.1-2021.06.20-000186",
      "managed" : true,
      "policy" : "existing-indices",
      "lifecycle_date_millis" : 1624267391799,
      "age" : "333.43d",
      "phase" : "cold",
      "phase_time_millis" : 1653075872766,
      "action" : "migrate",
      "action_time_millis" : 1653075873338,
      "step" : "check-migration",
      "step_time_millis" : 1653075873877,
      "step_info" : {
        "message" : "[filebeat-7.11.1-2021.06.20-000186] lifecycle action [migrate] waiting for [1] shards to be moved to the [data_cold] tier (tier migration preference configuration is [data_cold,data_warm,data_hot])",
        "shards_left_to_allocate" : 1,
        "all_shards_active" : true,
        "number_of_replicas" : 0
      },
      "phase_execution" : {
        "policy" : "existing-indices",
        "phase_definition" : {
          "min_age" : "30d",
          "actions" : {
            "set_priority" : {
              "priority" : 0
            }
          }
        },
        "version" : 6,
        "modified_date_in_millis" : 1653075872068
      }
    }
  }
}

[lchan]

I removed allocation.require.data setting but the step right now is "completed" and the index remains at hot node....

PUT /filebeat-7.11.1-2021.06.20-000186/_settings
{
  "index.routing.allocation.include._tier_preference": null,
  "index.routing.allocation.require.data": null
}

{
  "indices" : {
    "filebeat-7.11.1-2021.06.20-000186" : {
      "index" : "filebeat-7.11.1-2021.06.20-000186",
      "managed" : true,
      "policy" : "existing-indices",
      "lifecycle_date_millis" : 1624267391799,
      "age" : "333.43d",
      "phase" : "cold",
      "phase_time_millis" : 1653075872766,
      "action" : "complete",
      "action_time_millis" : 1653076322051,
      "step" : "complete",
      "step_time_millis" : 1653076322051,
      "phase_execution" : {
        "policy" : "existing-indices",
        "phase_definition" : {
          "min_age" : "30d",
          "actions" : {
            "set_priority" : {
              "priority" : 0
            }
          }
        },
        "version" : 6,
        "modified_date_in_millis" : 1653075872068
      }
    }
  }
}