I am very new to ELK. I have setup kibana, elasticsearch, logstash and filebeat. Data can be received successfully and shown in kibana but I have problem with index lifecycle management. There is always error like this: illegal_argument_exception: index.lifecycle.rollover_alias [filebeat-7.6.1] does not point to index [filebeat-7.6.1-2020.04.15]
I have tried to create new index myself: filebeat-7.6.1-2020.04.15-000001 and set it as write index and apply ilm but no data go into this index. The data goes into the old index. But I find out the index i create can roll up and delete itself as i want(so filebeat-7.6.1-2020.04.15-000001 is no longer exist and the existing one is filebeat-7.6.1-2020.04.15-000006 ). I really not sure what I miss. What i want to accomplish to build an index that can roll up each day, data goes into that index and it will be deleted itself a few days later.
Here is the setting of original index
{
"filebeat-7.6.1-2020.04.15" : {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "filebeat",
"rollover_alias" : "filebeat-7.6.1"
},
"mapping" : {
"total_fields" : {
"limit" : "10000"
}
},
"refresh_interval" : "5s",
"number_of_shards" : "1",
"provided_name" : "filebeat-7.6.1-2020.04.15",
"query" : {
"default_field" : [
..........
..........
]
},
"creation_date" : "1586943253337",
"number_of_replicas" : "1",
"uuid" : "nGw2BjyqTC6HODLTRRka1w",
"version" : {
"created" : "7060199"
}
}
}
}
}
Here is the setting of alias for the original index
{
"filebeat-7.6.1-2020.04.15" : {
"aliases" : { }
}
}
Here is the ilm setting for the original index
{
"indices" : {
"filebeat-7.6.1-2020.04.15" : {
"index" : "filebeat-7.6.1-2020.04.15",
"managed" : true,
"policy" : "filebeat",
"lifecycle_date_millis" : 1586943253337,
"age" : "5.91d",
"phase" : "hot",
"phase_time_millis" : 1587453505620,
"action" : "rollover",
"action_time_millis" : 1586943259018,
"step" : "ERROR",
"step_time_millis" : 1587454105616,
"failed_step" : "check-rollover-ready",
"is_auto_retryable_error" : true,
"failed_step_retry_count" : 8963,
"step_info" : {
"type" : "illegal_argument_exception",
"reason" : "index.lifecycle.rollover_alias [filebeat-7.6.1] does not point to index [filebeat-7.6.1-2020.04.15]",
"stack_trace" : """java.lang.IllegalArgumentException: index.lifecycle.rollover_alias [filebeat-7.6.1] does not point to index [filebeat-7.6.1-2020.04.15]
at org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep.evaluateCondition(WaitForRolloverReadyStep.java:104)
at org.elasticsearch.xpack.ilm.IndexLifecycleRunner.runPeriodicStep(IndexLifecycleRunner.java:152)
at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggerPolicies(IndexLifecycleService.java:328)
at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggered(IndexLifecycleService.java:266)
at org.elasticsearch.xpack.core.scheduler.SchedulerEngine.notifyListeners(SchedulerEngine.java:183)
at org.elasticsearch.xpack.core.scheduler.SchedulerEngine$ActiveSchedule.run(SchedulerEngine.java:211)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:830)
"""
},
"phase_execution" : {
"policy" : "filebeat",
"phase_definition" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "1d"
}
}
},
"version" : 7,
"modified_date_in_millis" : 1586955921457
}
}
}
}
Here is the setting of the index I create:
{
"filebeat-7.6.1-2020.04.15-000006" : {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "filebeat",
"rollover_alias" : "filebeat-7.6.1"
},
"mapping" : {
"total_fields" : {
"limit" : "10000"
}
},
"refresh_interval" : "5s",
"number_of_shards" : "1",
"provided_name" : "filebeat-7.6.1-2020.04.15-000006",
"query" : {
"default_field" : [
.............
..............
]
},
"creation_date" : "1587390505607",
"number_of_replicas" : "1",
"uuid" : "3sHTKpJjQ3OkW9IOxM1t5A",
"version" : {
"created" : "7060199"
}
}
}
}
}
The alias setting for the index i create
{
"filebeat-7.6.1-2020.04.15-000006" : {
"aliases" : {
"filebeat-7.6.1" : {
"is_write_index" : true
}
}
}
}
the ilm setting for the index i create
{
"indices" : {
"filebeat-7.6.1-2020.04.15-000006" : {
"index" : "filebeat-7.6.1-2020.04.15-000006",
"managed" : true,
"policy" : "filebeat",
"lifecycle_date_millis" : 1587390505607,
"age" : "17.82h",
"phase" : "hot",
"phase_time_millis" : 1587390506056,
"action" : "rollover",
"action_time_millis" : 1587391106375,
"step" : "check-rollover-ready",
"step_time_millis" : 1587391106375,
"phase_execution" : {
"policy" : "filebeat",
"phase_definition" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "1d"
}
}
},
"version" : 7,
"modified_date_in_millis" : 1586955921457
}
}
}
}
`````````````