Hi, i am working on Elasticsearch version 7.7.0 with logstash and creating indices with ILM and rolling index aliases. I have defined ILM like:
{
"weekly_rollover-4w_retention" : {
"version" : 2,
"modified_date" : "2020-10-15T09:22:19.049Z",
"policy" : {
"phases" : {
"hot" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "7d"
}
}
},
"delete" : {
"min_age" : "28d",
"actions" : {
"delete" : { }
}
},
"warm" : {
"min_age" : "7d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
}
}
}
}
}
}
}
And In my index template i define:
"lifecycle": {
"name": "weekly_rollover-4w_retention",
"parse_origination_date": "true",
"rollover_alias": "public-api"
},
I am trying to have date in index name corresponding to date when index is rolled. With absolutely same settings most of index is working, but i found that one is rolling but not changing that date, so it's origination_date is obsolete:
{
"indices" : {
"public-api-2020.11.06-000025" : {
"index" : "public-api-2020.11.06-000025",
"managed" : true,
"policy" : "weekly_rollover-4w_retention",
"lifecycle_date_millis" : 1604620800000,
"age" : "129.46d",
"phase" : "hot",
"phase_time_millis" : 1615560673691,
"action" : "rollover",
"action_time_millis" : 1615561272448,
"step" : "check-rollover-ready",
"step_time_millis" : 1615561272448,
"phase_execution" : {
"policy" : "weekly_rollover-4w_retention",
"phase_definition" : {
"min_age" : "0ms",
"actions" : {
"rollover" : {
"max_size" : "50gb",
"max_age" : "7d"
}
}
},
"version" : 2,
"modified_date_in_millis" : 1602753739049
}
}
}
}
From logs i can see that rolling is working, but have no clue why it's creating new index with bad date:
{"type": "server", "timestamp": "2021-03-12T14:51:11,051Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"check-rollover-ready\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"attempt-rollover\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:11,634Z", "level": "INFO", "component": "o.e.c.m.MetaDataCreateIndexService", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "[public-api-2020.11.06-000025] creating index, cause [rollover_index], templates [public-api], shards [1]/[1], mappings [_doc]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:12,309Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [null] to [{\"phase\":\"new\",\"action\":\"complete\",\"name\":\"complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:12,485Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"attempt-rollover\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"wait-for-active-shards\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:13,185Z", "level": "INFO", "component": "o.e.c.r.a.AllocationService", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[public-api-2020.11.06-000025][0]]]).", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:13,691Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"new\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-indexing-complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:14,139Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"wait-for-active-shards\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"update-rollover-lifecycle-date\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:14,141Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"update-rollover-lifecycle-date\"}] to [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"set-indexing-complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:14,745Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-indexing-complete\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-follow-shard-tasks\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:14,955Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"set-indexing-complete\"}] to [{\"phase\":\"hot\",\"action\":\"complete\",\"name\":\"complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:15,248Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"hot\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"readonly\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:15,818Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"readonly\"}] to [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"forcemerge\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T14:51:16,044Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"forcemerge\"}] to [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"segment-count\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T15:01:11,063Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"forcemerge\",\"name\":\"segment-count\"}] to [{\"phase\":\"warm\",\"action\":\"complete\",\"name\":\"complete\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T15:01:11,286Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"wait-for-follow-shard-tasks\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"pause-follower-index\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T15:01:11,528Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000024] from [{\"phase\":\"warm\",\"action\":\"complete\",\"name\":\"complete\"}] to [{\"phase\":\"delete\",\"action\":\"delete\",\"name\":\"wait-for-shard-history-leases\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
{"type": "server", "timestamp": "2021-03-12T15:01:11,733Z", "level": "INFO", "component": "o.e.x.i.IndexLifecycleTransition", "cluster.name": "elastic-stack-dev", "node.name": "eslog2", "message": "moving index [public-api-2020.11.06-000025] from [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"pause-follower-index\"}] to [{\"phase\":\"hot\",\"action\":\"unfollow\",\"name\":\"close-follower-index\"}] in policy [weekly_rollover-4w_retention]", "cluster.uuid": "Gi_9oOBxSbatR7qy7IaXBg", "node.id": "7qf1BhNaQFydAHoYaOtWzg" }
....
Data are send via logstash with simple configuration:
elasticsearch {
hosts => ["eslog4:9200", "eslog5:9200"]
timeout => 5
manage_template => false
ilm_enabled => true
ilm_rollover_alias => "sms-admin"
}
Wierd is that on different index with different name but absolutely same settings it's working without any problems, indices are rolled and have correct date in their name.
Thanks