Ilm in elstic cloud does not move the indexes from hot nodes to warm nodes

Hello,

I have set ilm rules to move hot indexes to warm indexes and move them into warm node. Now all my indexes are located on the hot nodes and eate up all my disk while the warm node are free and not doing anything having a bunch of disk available. Is here anyone who can help me fixing the issues?

My ilm rules look like this:

_GET _ilm/policy
{
  "apm-rollover-30-days" : {
    "version" : 2,
    "modified_date" : "2021-04-22T18:03:18.774Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            },
            "set_priority" : {
              "priority" : 100
            }
          }
        },
        "warm" : {
          "min_age" : "1d",
          "actions" : {
            "readonly" : { },
            "set_priority" : {
              "priority" : 50
            }
          }
        },
        "cold" : {
          "min_age" : "0d",
          "actions" : {
            "searchable_snapshot" : {
              "snapshot_repository" : "found-snapshots",
              "force_merge_index" : true
            },
            "set_priority" : {
              "priority" : 0
            }
          }
        }
      }
    }
  },
  "filebeat" : {
    "version" : 4,
    "modified_date" : "2021-04-22T18:33:01.898Z",
    "policy" : {
      "phases" : {
        "warm" : {
          "min_age" : "1d",
          "actions" : {
            "allocate" : {
              "number_of_replicas" : 1,
              "include" : { },
              "exclude" : { },
              "require" : { }
            },
            "readonly" : { },
            "set_priority" : {
              "priority" : 50
            }
          }
        },
        "cold" : {
          "min_age" : "10d",
          "actions" : {
            "searchable_snapshot" : {
              "snapshot_repository" : "found-snapshots",
              "force_merge_index" : true
            },
            "set_priority" : {
              "priority" : 10
            }
          }
        },
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_age" : "1d"
            }
          }
        },
        "delete" : {
          "min_age" : "365d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  },
  "ilm-history-ilm-policy" : {
    "version" : 1,
    "modified_date" : "2021-04-14T18:58:02.150Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            }
          }
        },
        "delete" : {
          "min_age" : "90d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  },
  "kibana-event-log-policy" : {
    "version" : 1,
    "modified_date" : "2021-04-14T18:58:41.007Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            }
          }
        },
        "delete" : {
          "min_age" : "90d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  },
  "logs" : {
    "version" : 1,
    "modified_date" : "2021-04-22T18:03:28.192Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            }
          }
        }
      }
    }
  },
  "metricbeat" : {
    "version" : 4,
    "modified_date" : "2021-04-22T18:36:09.593Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_age" : "1d"
            }
          }
        },
        "warm" : {
          "min_age" : "2d",
          "actions" : {
            "set_priority" : {
              "priority" : 50
            }
          }
        },
        "cold" : {
          "min_age" : "10d",
          "actions" : {
            "searchable_snapshot" : {
              "snapshot_repository" : "found-snapshots",
              "force_merge_index" : true
            },
            "set_priority" : {
              "priority" : 0
            }
          }
        }
      }
    }
  },
  "metrics" : {
    "version" : 1,
    "modified_date" : "2021-04-22T18:03:38.791Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            }
          }
        }
      }
    }
  },
  "ml-size-based-ilm-policy" : {
    "version" : 1,
    "modified_date" : "2021-04-14T18:58:00.013Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb"
            }
          }
        }
      }
    }
  },
  "slm-history-ilm-policy" : {
    "version" : 1,
    "modified_date" : "2021-04-14T18:58:02.199Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            }
          }
        },
        "delete" : {
          "min_age" : "90d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  },
  "synthetics" : {
    "version" : 1,
    "modified_date" : "2021-04-14T18:58:00.120Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            }
          }
        }
      }
    }
  },
  "watch-history-ilm-policy" : {
    "version" : 1,
    "modified_date" : "2021-04-14T18:58:02.099Z",
    "policy" : {
      "phases" : {
        "delete" : {
          "min_age" : "7d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  }
}

The cluster in not in healty state beacuse of this. Please help me how to fix it.

Can you share a screenshot of your config in Elastic Cloud page that looks like this;

I moved the move to warm nodes for 1 day after the initial roleout and it started to work.

1 Like

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.